diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index 0696758780b..f1ee94e2ff1 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -2,7 +2,12 @@ module internal FSharp.Compiler.OptimizeInputs +open System.Collections.Generic +open System.Diagnostics open System.IO +open System.Threading +open FSharp.Compiler.Optimizer +open FSharp.Compiler.Service.Driver.OptimizeTypes open Internal.Utilities.Library open FSharp.Compiler open FSharp.Compiler.AbstractIL.IL @@ -48,6 +53,229 @@ let GetInitialOptimizationEnv (tcImports: TcImports, tcGlobals: TcGlobals) = let optEnv = List.fold (AddExternalCcuToOptimizationEnv tcGlobals) optEnv ccuinfos optEnv +let collectResults (inputs: CollectorInputs) : CollectorOutputs = + let files = + inputs + |> Array.map (fun {Phase1 = phase1; Phase2 = _phase2; Phase3 = phase3} -> + let (_, _, implFileOptData, _), optimizeDuringCodeGen = phase1 + let _, implFile = phase3 + let implFile = + { + ImplFile = implFile + OptimizeDuringCodeGen = optimizeDuringCodeGen + } + implFile, implFileOptData + ) + + let lastFilePhase1Env = + inputs + |> Array.last + |> fun {Phase1 = phase1} -> + let (optEnvPhase1, _, _, _), _ = phase1 + optEnvPhase1 + + files, lastFilePhase1Env.Full + +type Phase = + | Phase1 + | Phase2 + | Phase3 + +type FilePhaseFuncs = Phase1Fun * Phase2Fun * Phase3Fun +type FileResults = + { + mutable Phase1: Phase1Res option + mutable Phase2: Phase2Res option + mutable Phase3: Phase3Res option + } + with + member this.HasResult (phase: Phase) = + match phase with + | Phase.Phase1 -> this.Phase1 |> Option.isSome + | Phase.Phase2 -> this.Phase2 |> Option.isSome + | Phase.Phase3 -> this.Phase3 |> Option.isSome + static member Empty = + { + Phase1 = None + Phase2 = None + Phase3 = None + } + +type WorkItem = + | Phase1 of Phase1Inputs + | Phase2 of Phase2Inputs + | Phase3 of Phase3Inputs + +type Idx = int +type Node = + { + Idx: Idx + Phase: Phase + } + with override this.ToString() = $"[{this.Idx}-{this.Phase}]" + + +let getPhase1Res (p: FileResults) = + p.Phase1 + |> Option.get + |> fun ((env, _, _, hidden), _) -> env, hidden + +let getPhase2Res (p: FileResults) = + p.Phase2 + |> Option.get + +let getPhase3Res (p: FileResults) = + p.Phase3 + |> Option.get + |> fun (env, _) -> env + +let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFuncs) (files: CheckedImplFile[]) : CollectorOutputs = + // Schedule File1-Phase1 + let firstNode = { Idx = 0; Phase = Phase.Phase1 } + + let results = + files + |> Array.map (fun _ -> FileResults.Empty) + + let _lock = obj() + let nodeCanBeProcessed ({Idx = idx; Phase = phase}) : bool = + lock (_lock) (fun () -> + let previousFileReady = + if idx = 0 then true else results[idx-1].HasResult phase + let previousPhase = + match phase with + | Phase.Phase1 -> None + | Phase.Phase2 -> Some Phase.Phase1 + | Phase.Phase3 -> Some Phase.Phase2 + let previousPhaseReady = + match previousPhase with + | Some previousPhase -> results[idx].HasResult previousPhase + | None -> true + previousFileReady && previousPhaseReady + ) + + let visited = HashSet() + + let worker ({Idx = idx; Phase = phase} as node : Node) : Node[] = + let notPreviouslyVisited = + lock (_lock) (fun () -> + visited.Add node + ) + if notPreviouslyVisited = false then [||] + else + let res = results[idx] + let file = files[idx] + let previous = if idx > 0 then Some results[idx-1] else None + let hidingInfo0 = SignatureHidingInfo.Empty + + let getPhase1Res (p: FileResults) = + p.Phase1 + |> Option.get + |> fun ((env, file, _, hidden), _) -> env, file, hidden + + match phase with + | Phase.Phase1 -> + // take env from previous file + let env, _, hidingInfo = + previous + |> Option.map getPhase1Res + |> Option.defaultValue ({Delta=env0; Full=env0}, file, {Delta=hidingInfo0; Full=hidingInfo0}) + let inputs = env.Full, hidingInfo.Full, file + let phase1Res = phase1 inputs + res.Phase1 <- Some phase1Res + + // Schedule Phase2 + let phase2Node = { Idx = idx; Phase = Phase.Phase2 } + seq { + yield phase2Node + if idx < files.Length-1 then yield { Idx = idx + 1; Phase = Phase.Phase1 } + } + |> Seq.toArray + + | Phase.Phase2 -> + // take env from previous file + let env = + previous + |> Option.map getPhase2Res + |> Option.map fst + |> Option.map (fun daf -> daf.Full) + |> Option.defaultValue env0 + let _optEnv, file, hidingInfo = + res + |> getPhase1Res + let inputs = env, hidingInfo.Full, file + let phase2Res = phase2 inputs + res.Phase2 <- Some phase2Res + + seq { + // Schedule Phase3 + let phase3Node = { Idx = idx; Phase = Phase.Phase3 } + yield phase3Node + // Schedule Phase2 for the next file if it exists + if idx < files.Length-1 then yield { Idx = idx + 1; Phase = Phase.Phase2 } + } + |> Seq.toArray + + | Phase.Phase3 -> + // take env from previous file + let env = + previous + |> Option.map getPhase3Res + |> Option.map (fun daf -> daf.Full) + |> Option.defaultValue env0 + // impl file + let _, file = + res + |> getPhase2Res + let hidingInfo = + res + |> getPhase1Res + |> fun (_a,_b,c) -> c.Full + let inputs = env, hidingInfo, file + let phase3Res = phase3 inputs + res.Phase3 <- Some phase3Res + + seq { + // Schedule Phase3 for the next file if it exists + if idx < files.Length-1 then yield { Idx = idx + 1; Phase = Phase.Phase3 } + } + |> Seq.toArray + |> fun nodes -> + nodes + |> Array.filter nodeCanBeProcessed + + ParallelTypeCheckingTests.Parallel.processInParallel + [|firstNode|] + worker + 10 + (fun _ -> visited.Count >= files.Length * 3) + (CancellationToken.None) + (fun node -> node.ToString()) + + Debug.Assert(visited.Count = files.Length * 3) + + let results = + results + |> Array.mapi (fun i {Phase1 = phase1; Phase2 = phase2; Phase3 = phase3} -> + match phase1, phase2, phase3 with + | Some phase1, Some phase2, Some phase3 -> {FileResultsComplete.Phase1 = phase1; Phase2 = phase2; Phase3 = phase3} + | _ -> failwith $"Unexpected lack of results for file [{i}]" + ) + let collected = results |> collectResults + collected + +type Goer = IReadOnlyDictionary -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs + +let mutable goer: Goer option = None + +[] +type OptimizerMode = + | Sequential + | PartiallyParallel + | GraphBased + +let mutable optimizerMode: OptimizerMode = OptimizerMode.Sequential + let ApplyAllOptimizations ( tcConfig: TcConfig, @@ -90,20 +318,68 @@ let ApplyAllOptimizations reportingPhase = true } - let results, (optEnvFirstLoop, _, _, _) = - ((optEnv0, optEnv0, optEnv0, SignatureHidingInfo.Empty), implFiles) - - ||> List.mapFold (fun (optEnvFirstLoop, optEnvExtraLoop, optEnvFinalSimplify, hidden) implFile -> - - //ReportTime tcConfig ("Initial simplify") - let (optEnvFirstLoop, implFile, implFileOptData, hidden), optimizeDuringCodeGen = + + let env0 = optEnv0 + + let envToDaf (env: IncrementalOptimizationEnv) (env0: IncrementalOptimizationEnv) = + { + Full = env + Delta = subtractEnv env env0 + } + + let hiddenToDaf (env: SignatureHidingInfo) (env0: SignatureHidingInfo) = + { + Full = env + Delta = subtractHidingInfo env env0 + } + + let phase1 (env: IncrementalOptimizationEnv, hidden: SignatureHidingInfo, implFile: CheckedImplFile) : Phase1Res = + //ReportTime tcConfig ("Initial simplify") + let (a,b,c,d), e = Optimizer.OptimizeImplFile( + optSettings, + ccu, + tcGlobals, + tcVal, + importMap, + env, + isIncrementalFragment, + tcConfig.fsiMultiAssemblyEmit, + tcConfig.emitTailcalls, + hidden, + implFile + ) + let a = envToDaf a env + let d = hiddenToDaf d hidden + (a,b,c,d),e + + let phase2 (env: IncrementalOptimizationEnv, hidden: SignatureHidingInfo, implFile: CheckedImplFile) : Phase2Res = + let implFile = LowerLocalMutables.TransformImplFile tcGlobals importMap implFile + + // Only do this on the first pass! + let optSettings = + { optSettings with + abstractBigTargets = false + reportingPhase = false + } +// TODO Uncomment +// #if DEBUG +// if tcConfig.showOptimizationData then +// dprintf +// "Optimization implFileOptData:\n%s\n" +// (LayoutRender.showL (Display.squashTo 192 (Optimizer.moduleInfoL tcGlobals implFileOptData))) +// #endif + + if tcConfig.extraOptimizationIterations > 0 then + + //ReportTime tcConfig ("Extra simplification loop") + let (optEnvExtraLoop, implFile, _, _), _ = Optimizer.OptimizeImplFile( optSettings, ccu, tcGlobals, tcVal, importMap, - optEnvFirstLoop, + env, isIncrementalFragment, tcConfig.fsiMultiAssemblyEmit, tcConfig.emitTailcalls, @@ -111,33 +387,94 @@ let ApplyAllOptimizations implFile ) - let implFile = LowerLocalMutables.TransformImplFile tcGlobals importMap implFile - - // Only do this on the first pass! - let optSettings = - { optSettings with - abstractBigTargets = false - reportingPhase = false - } -#if DEBUG - if tcConfig.showOptimizationData then - dprintf - "Optimization implFileOptData:\n%s\n" - (LayoutRender.showL (Display.squashTo 192 (Optimizer.moduleInfoL tcGlobals implFileOptData))) -#endif - - let implFile, optEnvExtraLoop = - if tcConfig.extraOptimizationIterations > 0 then + //PrintWholeAssemblyImplementation tcConfig outfile (sprintf "extra-loop-%d" n) implFile + envToDaf optEnvExtraLoop env, implFile + else + {Delta = IncrementalOptimizationEnv.Empty; Full = env}, implFile + + let phase3 (env: IncrementalOptimizationEnv, hidden: SignatureHidingInfo, implFile: CheckedImplFile) : Phase3Res = + // Only do this on the first pass! + let optSettings = + { optSettings with + abstractBigTargets = false + reportingPhase = false + } + + let implFile = + if tcConfig.doDetuple then + //ReportTime tcConfig ("Detupled optimization") + let implFile = implFile |> Detuple.DetupleImplFile ccu tcGlobals + //PrintWholeAssemblyImplementation tcConfig outfile "post-detuple" implFile + implFile + else + implFile + + let implFile = + if tcConfig.doTLR then + implFile + |> InnerLambdasToTopLevelFuncs.MakeTopLevelRepresentationDecisions ccu tcGlobals + else + implFile + + let implFile = LowerCalls.LowerImplFile tcGlobals implFile + + if tcConfig.doFinalSimplify then + + //ReportTime tcConfig ("Final simplify pass") + let (optEnvFinalSimplify, implFile, _, _), _ = + Optimizer.OptimizeImplFile( + optSettings, + ccu, + tcGlobals, + tcVal, + importMap, + env, + isIncrementalFragment, + tcConfig.fsiMultiAssemblyEmit, + tcConfig.emitTailcalls, + hidden, + implFile + ) - //ReportTime tcConfig ("Extra simplification loop") - let (optEnvExtraLoop, implFile, _, _), _ = + //PrintWholeAssemblyImplementation tcConfig outfile "post-rec-opt" implFile + envToDaf optEnvFinalSimplify env, implFile + else + {Delta = IncrementalOptimizationEnv.Empty; Full = env}, implFile + + let results, optEnvFirstLoop = + match optimizerMode with + | OptimizerMode.GraphBased -> + let graph = ParseAndCheckInputs.graph + // graph + // |> Seq.iter (fun (KeyValue(f, deps)) -> + // let d = System.String.Join(",", deps) + // printfn $"{f} - {d}" + // ) + let goer = goer.Value + let a, b = + goer graph env0 (phase1, phase2, phase3) (implFiles |> List.toArray) + a |> Array.toList, b + | OptimizerMode.PartiallyParallel -> + let a, b = + go env0 (phase1, phase2, phase3) (implFiles |> List.toArray) + a |> Array.toList, b + | OptimizerMode.Sequential -> + let results, (optEnvFirstLoop, _, _, _) = + ((optEnv0, optEnv0, optEnv0, SignatureHidingInfo.Empty), implFiles) + + ||> List.mapFold (fun (optEnvFirstLoop, optEnvExtraLoop, optEnvFinalSimplify, hidden) implFile -> + + // Phase 1 + + //ReportTime tcConfig ("Initial simplify") + let (optEnvFirstLoop, implFile, implFileOptData, hidden), optimizeDuringCodeGen = Optimizer.OptimizeImplFile( optSettings, ccu, tcGlobals, tcVal, importMap, - optEnvExtraLoop, + optEnvFirstLoop, isIncrementalFragment, tcConfig.fsiMultiAssemblyEmit, tcConfig.emitTailcalls, @@ -145,60 +482,108 @@ let ApplyAllOptimizations implFile ) - //PrintWholeAssemblyImplementation tcConfig outfile (sprintf "extra-loop-%d" n) implFile - implFile, optEnvExtraLoop - else - implFile, optEnvExtraLoop - - let implFile = - if tcConfig.doDetuple then - //ReportTime tcConfig ("Detupled optimization") - let implFile = implFile |> Detuple.DetupleImplFile ccu tcGlobals - //PrintWholeAssemblyImplementation tcConfig outfile "post-detuple" implFile - implFile - else - implFile - - let implFile = - if tcConfig.doTLR then - implFile - |> InnerLambdasToTopLevelFuncs.MakeTopLevelRepresentationDecisions ccu tcGlobals - else - implFile - - let implFile = LowerCalls.LowerImplFile tcGlobals implFile - let implFile, optEnvFinalSimplify = - if tcConfig.doFinalSimplify then - - //ReportTime tcConfig ("Final simplify pass") - let (optEnvFinalSimplify, implFile, _, _), _ = - Optimizer.OptimizeImplFile( - optSettings, - ccu, - tcGlobals, - tcVal, - importMap, - optEnvFinalSimplify, - isIncrementalFragment, - tcConfig.fsiMultiAssemblyEmit, - tcConfig.emitTailcalls, - hidden, + // Phase 2 + + let implFile = LowerLocalMutables.TransformImplFile tcGlobals importMap implFile + + // Only do this on the first pass! + let optSettings = + { optSettings with + abstractBigTargets = false + reportingPhase = false + } + #if DEBUG + if tcConfig.showOptimizationData then + dprintf + "Optimization implFileOptData:\n%s\n" + (LayoutRender.showL (Display.squashTo 192 (Optimizer.moduleInfoL tcGlobals implFileOptData))) + #endif + + let implFile, optEnvExtraLoop = + if tcConfig.extraOptimizationIterations > 0 then + + //ReportTime tcConfig ("Extra simplification loop") + let (optEnvExtraLoop, implFile, _, _), _ = + Optimizer.OptimizeImplFile( + optSettings, + ccu, + tcGlobals, + tcVal, + importMap, + optEnvExtraLoop, + isIncrementalFragment, + tcConfig.fsiMultiAssemblyEmit, + tcConfig.emitTailcalls, + hidden, + implFile + ) + + //PrintWholeAssemblyImplementation tcConfig outfile (sprintf "extra-loop-%d" n) implFile + implFile, optEnvExtraLoop + else + implFile, optEnvExtraLoop + + + // Phase 3 + + // Only do this on the first pass! + let optSettings = + { optSettings with + abstractBigTargets = false + reportingPhase = false + } + + let implFile = + if tcConfig.doDetuple then + //ReportTime tcConfig ("Detupled optimization") + let implFile = implFile |> Detuple.DetupleImplFile ccu tcGlobals + //PrintWholeAssemblyImplementation tcConfig outfile "post-detuple" implFile + implFile + else implFile - ) - - //PrintWholeAssemblyImplementation tcConfig outfile "post-rec-opt" implFile - implFile, optEnvFinalSimplify - else - implFile, optEnvFinalSimplify - let implFile = - { - ImplFile = implFile - OptimizeDuringCodeGen = optimizeDuringCodeGen - } + let implFile = + if tcConfig.doTLR then + implFile + |> InnerLambdasToTopLevelFuncs.MakeTopLevelRepresentationDecisions ccu tcGlobals + else + implFile - (implFile, implFileOptData), (optEnvFirstLoop, optEnvExtraLoop, optEnvFinalSimplify, hidden)) + let implFile = LowerCalls.LowerImplFile tcGlobals implFile + + let implFile, optEnvFinalSimplify = + if tcConfig.doFinalSimplify then + + //ReportTime tcConfig ("Final simplify pass") + let (optEnvFinalSimplify, implFile, _, _), _ = + Optimizer.OptimizeImplFile( + optSettings, + ccu, + tcGlobals, + tcVal, + importMap, + optEnvFinalSimplify, + isIncrementalFragment, + tcConfig.fsiMultiAssemblyEmit, + tcConfig.emitTailcalls, + hidden, + implFile + ) + + //PrintWholeAssemblyImplementation tcConfig outfile "post-rec-opt" implFile + implFile, optEnvFinalSimplify + else + implFile, optEnvFinalSimplify + + let implFile = + { + ImplFile = implFile + OptimizeDuringCodeGen = optimizeDuringCodeGen + } + + (implFile, implFileOptData), (optEnvFirstLoop, optEnvExtraLoop, optEnvFinalSimplify, hidden)) + results, optEnvFirstLoop let implFiles, implFileOptDatas = List.unzip results let assemblyOptData = Optimizer.UnionOptimizationInfos implFileOptDatas diff --git a/src/Compiler/Driver/OptimizeInputs.fsi b/src/Compiler/Driver/OptimizeInputs.fsi index d5c731ba05d..b03bed636d8 100644 --- a/src/Compiler/Driver/OptimizeInputs.fsi +++ b/src/Compiler/Driver/OptimizeInputs.fsi @@ -12,6 +12,8 @@ open FSharp.Compiler.Import open FSharp.Compiler.Optimizer open FSharp.Compiler.TcGlobals open FSharp.Compiler.TypedTree +open System.Collections.Generic +open FSharp.Compiler.Service.Driver.OptimizeTypes val GetGeneratedILModuleName: CompilerTarget -> string -> string @@ -49,3 +51,14 @@ val GenerateIlxCode: val NormalizeAssemblyRefs: CompilationThreadToken * ILGlobals * TcImports -> (ILScopeRef -> ILScopeRef) val GetGeneratedILModuleName: CompilerTarget -> string -> string + +type FilePhaseFuncs = Phase1Fun * Phase2Fun * Phase3Fun +type Goer = IReadOnlyDictionary -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs +val mutable goer: Goer option + +type OptimizerMode = + | Sequential + | PartiallyParallel + | GraphBased + +val mutable optimizerMode: OptimizerMode \ No newline at end of file diff --git a/src/Compiler/Driver/OptimizeTypes.fs b/src/Compiler/Driver/OptimizeTypes.fs new file mode 100644 index 00000000000..d37024406c9 --- /dev/null +++ b/src/Compiler/Driver/OptimizeTypes.fs @@ -0,0 +1,92 @@ +module internal FSharp.Compiler.Service.Driver.OptimizeTypes + +open FSharp.Compiler +open FSharp.Compiler.Optimizer +open FSharp.Compiler.TcGlobals +open FSharp.Compiler.TypedTree +open FSharp.Compiler.TypedTreeOps + +type DeltaAndFull<'a> = + { + Delta: 'a + Full: 'a + } + +type OptimizeDuringCodeGen = bool -> Expr -> Expr +type OptimizeRes = + (DeltaAndFull * CheckedImplFile * ImplFileOptimizationInfo * DeltaAndFull) * OptimizeDuringCodeGen + +type Optimize = + OptimizationSettings * + CcuThunk * + TcGlobals * + ConstraintSolver.TcValF * + Import.ImportMap * + IncrementalOptimizationEnv * + bool * + bool * + bool * + SignatureHidingInfo * + CheckedImplFile -> + OptimizeRes + +type PhaseInputs = IncrementalOptimizationEnv * SignatureHidingInfo * CheckedImplFile + +type Phase1Inputs = PhaseInputs +type Phase1Res = OptimizeRes +type Phase1Fun = Phase1Inputs -> Phase1Res + +type Phase2Inputs = PhaseInputs +type Phase2Res = DeltaAndFull * CheckedImplFile +type Phase2Fun = Phase2Inputs -> Phase2Res + +type Phase3Inputs = PhaseInputs +type Phase3Res = DeltaAndFull * CheckedImplFile +type Phase3Fun = Phase3Inputs -> Phase3Res + +type Phase = + | Phase1 + | Phase2 + | Phase3 +module Phase = + let all = [|Phase1; Phase2; Phase3|] + let prev (phase: Phase) = + match phase with + | Phase1 -> None + | Phase2 -> Some Phase1 + | Phase3 -> Some Phase2 + let next (phase: Phase) = + match phase with + | Phase1 -> Some Phase2 + | Phase2 -> Some Phase3 + | Phase3 -> None + +type PhaseRes = + | Phase1 of Phase1Res + | Phase2 of Phase2Res + | Phase3 of Phase3Res + with + member x.Which = + match x with + | Phase1 _ -> Phase.Phase1 + | Phase2 _ -> Phase.Phase2 + | Phase3 _ -> Phase.Phase3 + member x.Get1() = + match x with + | Phase1 x -> x + | Phase2 _ + | Phase3 _ -> failwith $"Called {nameof(x.Get1)} but this is {x.Which}" + member x.Get2() = + match x with + | Phase2 x -> x + | Phase1 _ + | Phase3 _ -> failwith $"Called {nameof(x.Get2)} but this is {x.Which}" + +type FileResultsComplete = + { + Phase1: Phase1Res + Phase2: Phase2Res + Phase3: Phase3Res + } +type CollectorInputs = FileResultsComplete[] +type CollectorOutputs = (CheckedImplFileAfterOptimization * ImplFileOptimizationInfo)[] * IncrementalOptimizationEnv diff --git a/src/Compiler/Driver/Parallel.fs b/src/Compiler/Driver/Parallel.fs new file mode 100644 index 00000000000..dbe9394a63b --- /dev/null +++ b/src/Compiler/Driver/Parallel.fs @@ -0,0 +1,54 @@ +module ParallelTypeCheckingTests.Parallel + +#nowarn "1182" + +open System +open System.Collections.Concurrent +open System.Threading + +// TODO Could replace with MailboxProcessor+Tasks/Asyncs instead of BlockingCollection + Threads +// See http://www.fssnip.net/nX/title/Limit-degree-of-parallelism-using-an-agent +/// Process items in parallel, allow more work to be scheduled as a result of finished work, +/// limit parallelisation to 'parallelism' threads +let processInParallel + (firstItems: 'Item[]) + (work: 'Item -> 'Item[]) + (parallelism: int) + (shouldStop: int -> bool) + (ct: CancellationToken) + (_itemToString: 'Item -> string) + : unit = + let bc = new BlockingCollection<'Item>() + firstItems |> Array.iter bc.Add + let processedCountLock = Object() + let mutable processedCount = 0 + + let processItem item = + printfn $"[{Thread.CurrentThread.ManagedThreadId}] Processing {_itemToString item}" + let toSchedule = work item + printfn $"[{Thread.CurrentThread.ManagedThreadId}] Finished {_itemToString item}" + + let processedCount = + lock processedCountLock (fun () -> + processedCount <- processedCount + 1 + processedCount) + + let toScheduleString = + toSchedule |> Array.map _itemToString |> (fun names -> String.Join(", ", names)) + + printfn $"[{Thread.CurrentThread.ManagedThreadId}] Scheduling {toSchedule.Length} items: {toScheduleString}" + toSchedule |> Array.iter bc.Add + processedCount + + // TODO Could avoid workers with some semaphores + let workerWork () : unit = + for node in bc.GetConsumingEnumerable(ct) do + if not ct.IsCancellationRequested then // improve + let processedCount = processItem node + + if shouldStop processedCount then + bc.CompleteAdding() + + // TODO Do we need to handle cancellation given that workers do it already? + Array.Parallel.map workerWork (Array.init parallelism (fun _ -> ())) |> ignore + diff --git a/src/Compiler/Driver/ParseAndCheckInputs.fs b/src/Compiler/Driver/ParseAndCheckInputs.fs index 0bfc5e7a575..02eec82077b 100644 --- a/src/Compiler/Driver/ParseAndCheckInputs.fs +++ b/src/Compiler/Driver/ParseAndCheckInputs.fs @@ -759,29 +759,17 @@ let ParseInputFilesInParallel (tcConfig: TcConfig, lexResourceManager, sourceFil for fileName in sourceFiles do checkInputFile tcConfig fileName - // Order files to be parsed by size (descending). The idea is to process big files first, - // so that near the end when only some nodes are still processing items, it's the smallest items, - // which should reduce the period of time where only some nodes are busy. - // This requires some empirical evidence. - let sourceFiles = - sourceFiles - |> List.mapi (fun i f -> i, f) - |> List.sortBy (fun (_i, f) -> -FileInfo(f).Length) - let sourceFiles = List.zip sourceFiles isLastCompiland UseMultipleDiagnosticLoggers (sourceFiles, delayLogger, None) (fun sourceFilesWithDelayLoggers -> sourceFilesWithDelayLoggers - |> ListParallel.map (fun (((idx, fileName), isLastCompiland), delayLogger) -> + |> ListParallel.map (fun ((fileName, isLastCompiland), delayLogger) -> let directoryName = Path.GetDirectoryName fileName let input = parseInputFileAux (tcConfig, lexResourceManager, fileName, (isLastCompiland, isExe), delayLogger, retryLocked) - idx, (input, directoryName)) - // Bring back index-based order - |> List.sortBy fst - |> List.map snd) + (input, directoryName))) let ParseInputFilesSequential (tcConfig: TcConfig, lexResourceManager, sourceFiles, diagnosticsLogger: DiagnosticsLogger, retryLocked) = let isLastCompiland, isExe = sourceFiles |> tcConfig.ComputeCanContainEntryPoint @@ -1767,3 +1755,5 @@ let CheckClosedInputSet (ctok, checkForErrors, tcConfig: TcConfig, tcImports, tc tcState.Ccu.Deref.Contents <- ccuContents tcState, topAttrs, declaredImpls, tcEnvAtEndOfLastFile + +let mutable graph: System.Collections.Generic.IReadOnlyDictionary = null \ No newline at end of file diff --git a/src/Compiler/Driver/ParseAndCheckInputs.fsi b/src/Compiler/Driver/ParseAndCheckInputs.fsi index 708111d89ff..5ccaa14a331 100644 --- a/src/Compiler/Driver/ParseAndCheckInputs.fsi +++ b/src/Compiler/Driver/ParseAndCheckInputs.fsi @@ -228,3 +228,5 @@ val CheckOneInputAndFinish: tcState: TcState * input: ParsedInput -> Cancellable<(TcEnv * TopAttribs * CheckedImplFile list * ModuleOrNamespaceType list) * TcState> + +val mutable graph: System.Collections.Generic.IReadOnlyDictionary \ No newline at end of file diff --git a/src/Compiler/Driver/fsc.fs b/src/Compiler/Driver/fsc.fs index a3968cef5e3..66edbe55ab9 100644 --- a/src/Compiler/Driver/fsc.fs +++ b/src/Compiler/Driver/fsc.fs @@ -20,6 +20,7 @@ open System.Reflection open System.Text open System.Threading +open System.Threading.Tasks open Internal.Utilities open Internal.Utilities.Library open Internal.Utilities.Library.Extras @@ -823,13 +824,6 @@ let main3 ReportTime tcConfig "Encode Interface Data" let exportRemapping = MakeExportRemapping generatedCcu generatedCcu.Contents - let sigDataAttributes, sigDataResources = - try - EncodeSignatureData(tcConfig, tcGlobals, exportRemapping, generatedCcu, outfile, false) - with e -> - errorRecoveryNoRange e - exiter.Exit 1 - let metadataVersion = match tcConfig.metadataVersion with | Some v -> v @@ -837,35 +831,60 @@ let main3 match frameworkTcImports.DllTable.TryFind tcConfig.primaryAssembly.Name with | Some ib -> ib.RawMetadata.TryGetILModuleDef().Value.MetadataVersion | _ -> "" + + let (sigDataAttributes, sigDataResources), optimizedImpls, optDataResources = + let mutable sigDataAttributes: ILAttribute list = [] + let mutable sigDataResources : ILResource list = [] + let a1 = + async { + try + let sigDataAttributes2, sigDataResources2 = EncodeSignatureData(tcConfig, tcGlobals, exportRemapping, generatedCcu, outfile, false) + sigDataAttributes <- sigDataAttributes2 + sigDataResources <- sigDataResources2 + with e -> + errorRecoveryNoRange e + exiter.Exit 1 + } - let optimizedImpls, optDataResources = - // Perform optimization - use _ = UseBuildPhase BuildPhase.Optimize - - let optEnv0 = GetInitialOptimizationEnv(tcImports, tcGlobals) - - let importMap = tcImports.GetImportMap() - - let optimizedImpls, optimizationData, _ = - ApplyAllOptimizations( - tcConfig, - tcGlobals, - (LightweightTcValForUsingInBuildMethodCall tcGlobals), - outfile, - importMap, - false, - optEnv0, - generatedCcu, - typedImplFiles - ) - - AbortOnError(diagnosticsLogger, exiter) + let mutable optimizedImpls2 : CheckedAssemblyAfterOptimization option = None + let mutable optDataResources : ILResource list = [] + let a2 = + async { + // Perform optimization + use _ = UseBuildPhase BuildPhase.Optimize + + let optEnv0 = GetInitialOptimizationEnv(tcImports, tcGlobals) + + let importMap = tcImports.GetImportMap() + + let optimizedImpls, optimizationData, _ = + ApplyAllOptimizations( + tcConfig, + tcGlobals, + (LightweightTcValForUsingInBuildMethodCall tcGlobals), + outfile, + importMap, + false, + optEnv0, + generatedCcu, + typedImplFiles + ) - // Encode the optimization data - ReportTime tcConfig ("Encoding OptData") + AbortOnError(diagnosticsLogger, exiter) - optimizedImpls, EncodeOptimizationData(tcGlobals, tcConfig, outfile, exportRemapping, (generatedCcu, optimizationData), false) + // Encode the optimization data + ReportTime tcConfig ("Encoding OptData") + optimizedImpls2 <- Some optimizedImpls + optDataResources <- EncodeOptimizationData(tcGlobals, tcConfig, outfile, exportRemapping, (generatedCcu, optimizationData), false) + } + + let t1 = a1 |> Async.StartAsTask + let t2 = a2 |> Async.StartAsTask + t1.Wait() + t2.Wait() + (sigDataAttributes, sigDataResources), optimizedImpls2.Value, optDataResources + // Pass on only the minimum information required for the next phase Args( ctok, diff --git a/src/Compiler/FSharp.Compiler.Service.fsproj b/src/Compiler/FSharp.Compiler.Service.fsproj index f479840dfbd..2298d7be5e7 100644 --- a/src/Compiler/FSharp.Compiler.Service.fsproj +++ b/src/Compiler/FSharp.Compiler.Service.fsproj @@ -387,6 +387,8 @@ + + diff --git a/src/Compiler/Optimize/Optimizer.fs b/src/Compiler/Optimize/Optimizer.fs index 54665570cc5..8a313f52e7e 100644 --- a/src/Compiler/Optimize/Optimizer.fs +++ b/src/Compiler/Optimize/Optimizer.fs @@ -441,6 +441,7 @@ type MethodEnv = override x.ToString() = "" + type IncrementalOptimizationEnv = { /// An identifier to help with name generation latestBoundId: Ident option @@ -481,6 +482,78 @@ type IncrementalOptimizationEnv = override x.ToString() = "" +let mergeMaps<'Key, 'Value when 'Key : comparison> (maps: Map<'Key, 'Value>[]) = + maps + |> Array.collect Map.toArray + |> Map.ofArray + +let mapDiff<'a,'b when 'a: comparison> (a: Map<'a,'b>) (b: Map<'a,'b>) : Map<'a,'b> = + a + // TODO What if a & b contain a key but with different values? Is that possible? + |> Seq.choose (fun (KeyValue(k, v)) -> + match b.TryFind k with + | Some _ -> None + | None -> Some (k, v) + ) + |> Map.ofSeq + +let subtractEnv (a: IncrementalOptimizationEnv) (b: IncrementalOptimizationEnv): IncrementalOptimizationEnv = + { + a with + //latestBoundId = env0.latestBoundId // Not used across files + dontInline = Zset.diff a.dontInline b.dontInline// sum + typarInfos = a.typarInfos |> List.skip b.typarInfos.Length // sum + //functionVal = None // Not used across files + dontSplitVars = ValMap(mapDiff a.dontSplitVars.Contents b.dontSplitVars.Contents) // sum + //disableMethodSplitting = false // not used across files + localExternalVals = mapDiff a.localExternalVals b.localExternalVals // sum + globalModuleInfos = mapDiff a.globalModuleInfos b.globalModuleInfos // sum + //methEnv = { pipelineCount = 0 } // Not used across files + } + +let subtractHidingInfo (a: SignatureHidingInfo) (b: SignatureHidingInfo) : SignatureHidingInfo = + { + SignatureHidingInfo.HiddenTycons = Zset.diff a.HiddenTycons b.HiddenTycons + HiddenTyconReprs = Zset.diff a.HiddenTyconReprs b.HiddenTyconReprs + HiddenVals = Zset.diff a.HiddenVals b.HiddenVals + HiddenRecdFields = Zset.diff a.HiddenRecdFields b.HiddenRecdFields + HiddenUnionCases = Zset.diff a.HiddenUnionCases b.HiddenUnionCases + } + +let mergeEnvs (env0: IncrementalOptimizationEnv) (envs: IncrementalOptimizationEnv[]): IncrementalOptimizationEnv = + let envs = Array.append [|env0|] envs + // TODO use a single HashSet for perf? + let dontInline = + envs + |> Array.collect (fun e -> e.dontInline.ToArray()) + |> fun xs -> Zset.Create (Int64.order, xs) + let typarInfos = envs |> Array.toList |> List.collect (fun e -> e.typarInfos) + let dontSplitVars = + envs + |> Seq.collect (fun e -> e.dontSplitVars.Contents |> Map.toSeq) + |> Seq.toArray + |> ValMap.OfArray' + let localExternalVals = + envs + |> Array.map (fun e -> e.localExternalVals) + |> mergeMaps + let globalModuleInfos = + envs + |> Array.map (fun e -> e.globalModuleInfos) + |> mergeMaps + { + env0 with + //latestBoundId = env0.latestBoundId // Not used across files + dontInline = dontInline// sum + typarInfos = typarInfos // sum + //functionVal = None // Not used across files + dontSplitVars = dontSplitVars // sum + //disableMethodSplitting = false // not used across files + localExternalVals = localExternalVals // sum + globalModuleInfos = globalModuleInfos // sum + //methEnv = { pipelineCount = 0 } // Not used across files + } + //------------------------------------------------------------------------- // IsPartialExprVal - is the expr fully known? //------------------------------------------------------------------------- @@ -567,9 +640,9 @@ let BindExternalLocalVal cenv (v: Val) vval env = let vval = if v.IsMutable then {vval with ValExprInfo=UnknownValue } else vval - let env = + let acc = match vval.ValExprInfo with - | UnknownValue -> env + | UnknownValue -> env | _ -> { env with localExternalVals=env.localExternalVals.Add (v.Stamp, vval) } // If we're compiling fslib then also bind the value as a non-local path to @@ -579,14 +652,16 @@ let BindExternalLocalVal cenv (v: Val) vval env = // v, dereferencing it to find the corresponding signature Val, and adding an entry for the signature val. // // A similar code path exists in ilxgen.fs for the tables of "representations" for values - let env = - if g.compilingFSharpCore then + let env = + // TODO We ignore the fact this isn't delta-based for now. + if g.compilingFSharpCore then + failwith "FSharpCore optimization not supported on this PoC branch" // Passing an empty remap is sufficient for FSharp.Core.dll because it turns out the remapped type signature can // still be resolved. match tryRescopeVal g.fslibCcu Remap.Empty v with | ValueSome vref -> BindValueForFslib vref.nlr v vval env | _ -> env - else env + else acc env let rec BindValsInModuleOrNamespace cenv (mval: LazyModuleInfo) env = @@ -4272,7 +4347,16 @@ and OptimizeModuleDefs cenv (env, bindInfosColl) defs = let defs, (env, bindInfosColl) = List.mapFold (OptimizeModuleContents cenv) (env, bindInfosColl) defs let defs, minfos = List.unzip defs (defs, UnionOptimizationInfos minfos), (env, bindInfosColl) - + +and mergeSignatureHidingInfos (a: SignatureHidingInfo) (b: SignatureHidingInfo) = + { + SignatureHidingInfo.HiddenTycons = Zset.union a.HiddenTycons b.HiddenTycons + HiddenTyconReprs = Zset.union a.HiddenTyconReprs b.HiddenTyconReprs + HiddenVals = Zset.union a.HiddenVals b.HiddenVals + HiddenRecdFields = Zset.union a.HiddenRecdFields b.HiddenRecdFields + HiddenUnionCases = Zset.union a.HiddenUnionCases b.HiddenUnionCases + } + and OptimizeImplFileInternal cenv env isIncrementalFragment fsiMultiAssemblyEmit hidden implFile = let g = cenv.g let (CheckedImplFile (qname, pragmas, signature, contents, hasExplicitEntryPoint, isScript, anonRecdTypes, namedDebugPointsForInlinedCode)) = implFile @@ -4298,7 +4382,8 @@ and OptimizeImplFileInternal cenv env isIncrementalFragment fsiMultiAssemblyEmit else // This optimizes and builds minfo w.r.t. the signature let mexprR, minfo = OptimizeModuleExprWithSig cenv env signature contents - let hidden = ComputeSignatureHidingInfoAtAssemblyBoundary signature hidden + let hiddenDelta = ComputeSignatureHidingInfoAtAssemblyBoundary signature SignatureHidingInfo.Empty + let hidden = mergeSignatureHidingInfos hidden hiddenDelta let minfoExternal = AbstractLazyModulInfoByHiding true hidden minfo let env = // In F# interactive multi-assembly mode, internals are not accessible in the 'env' used intra-assembly @@ -4311,7 +4396,7 @@ and OptimizeImplFileInternal cenv env isIncrementalFragment fsiMultiAssemblyEmit let implFileR = CheckedImplFile (qname, pragmas, signature, contentsR, hasExplicitEntryPoint, isScript, anonRecdTypes, namedDebugPointsForInlinedCode) - env, implFileR, minfo, hidden + env , implFileR, minfo, hidden /// Entry point let OptimizeImplFile (settings, ccu, tcGlobals, tcVal, importMap, optEnv, isIncrementalFragment, fsiMultiAssemblyEmit, emitTailcalls, hidden, mimpls) = diff --git a/src/Compiler/Optimize/Optimizer.fsi b/src/Compiler/Optimize/Optimizer.fsi index 76b7881f487..d57de0f7022 100644 --- a/src/Compiler/Optimize/Optimizer.fsi +++ b/src/Compiler/Optimize/Optimizer.fsi @@ -61,6 +61,12 @@ type CcuOptimizationInfo = LazyModuleInfo type IncrementalOptimizationEnv = static member Empty: IncrementalOptimizationEnv +val mergeEnvs : IncrementalOptimizationEnv -> IncrementalOptimizationEnv[] -> IncrementalOptimizationEnv + +val subtractEnv : IncrementalOptimizationEnv -> IncrementalOptimizationEnv -> IncrementalOptimizationEnv + +val subtractHidingInfo : SignatureHidingInfo -> SignatureHidingInfo -> SignatureHidingInfo + /// For building optimization environments incrementally val internal BindCcu: CcuThunk -> CcuOptimizationInfo -> IncrementalOptimizationEnv -> TcGlobals -> IncrementalOptimizationEnv diff --git a/src/Compiler/TypedTree/TypedTreeOps.fs b/src/Compiler/TypedTree/TypedTreeOps.fs index f57954f7bf5..a000c3e47a1 100644 --- a/src/Compiler/TypedTree/TypedTreeOps.fs +++ b/src/Compiler/TypedTree/TypedTreeOps.fs @@ -86,7 +86,8 @@ type ValMap<'T>(imap: StampMap<'T>) = member _.Remove (v: Val) = ValMap (imap.Remove(v.Stamp)) static member Empty = ValMap<'T> Map.empty member _.IsEmpty = imap.IsEmpty - static member OfList vs = (vs, ValMap<'T>.Empty) ||> List.foldBack (fun (x, y) acc -> acc.Add x y) + static member OfList vs = (vs, ValMap<'T>.Empty) ||> List.foldBack (fun (x, y) acc -> acc.Add x y) + static member OfArray' (vs: (Stamp * 'T)[]) = (vs, ValMap<'T>.Empty) ||> Array.foldBack (fun (x, y) acc -> ValMap (acc.Contents.Add(x, y))) //-------------------------------------------------------------------------- // renamings @@ -4641,6 +4642,13 @@ type SignatureHidingInfo = HiddenVals = Zset.empty valOrder HiddenRecdFields = Zset.empty recdFieldRefOrder HiddenUnionCases = Zset.empty unionCaseRefOrder } + + static member Union (a: SignatureHidingInfo) (b: SignatureHidingInfo): SignatureHidingInfo = + { HiddenTycons = Zset.union a.HiddenTycons b.HiddenTycons + HiddenTyconReprs = Zset.union a.HiddenTyconReprs b.HiddenTyconReprs + HiddenVals = Zset.union a.HiddenVals b.HiddenVals + HiddenRecdFields = Zset.union a.HiddenRecdFields b.HiddenRecdFields + HiddenUnionCases = Zset.union a.HiddenUnionCases b.HiddenUnionCases } let addValRemap v vNew tmenv = { tmenv with valRemap= tmenv.valRemap.Add v (mkLocalValRef vNew) } diff --git a/src/Compiler/TypedTree/TypedTreeOps.fsi b/src/Compiler/TypedTree/TypedTreeOps.fsi index 00d838e04d4..aeebd18033b 100755 --- a/src/Compiler/TypedTree/TypedTreeOps.fsi +++ b/src/Compiler/TypedTree/TypedTreeOps.fsi @@ -407,6 +407,8 @@ val mkExprAddrOfExpr: [] type ValMap<'T> = + new : StampMap<'T> -> ValMap<'T> + member Contents: StampMap<'T> member Item: Val -> 'T with get @@ -424,6 +426,8 @@ type ValMap<'T> = static member Empty: ValMap<'T> static member OfList: (Val * 'T) list -> ValMap<'T> + + static member OfArray': (Stamp * 'T)[] -> ValMap<'T> /// Mutable data structure mapping Val's to T based on stamp keys [] @@ -1269,6 +1273,8 @@ type SignatureHidingInfo = /// The empty table representing no hiding static member Empty: SignatureHidingInfo + + static member Union: SignatureHidingInfo -> SignatureHidingInfo -> SignatureHidingInfo /// Compute the remapping information implied by a signature being inferred for a particular implementation val ComputeRemappingFromImplementationToSignature: diff --git a/tests/FSharp.Test.Utilities/Compiler.fs b/tests/FSharp.Test.Utilities/Compiler.fs index bf769993930..f3cb5f62193 100644 --- a/tests/FSharp.Test.Utilities/Compiler.fs +++ b/tests/FSharp.Test.Utilities/Compiler.fs @@ -404,7 +404,7 @@ module rec Compiler = withOptionsHelper [ $"--nowarn:{warning}" ] "withNoWarn is only supported for F#" cUnit let withNoOptimize (cUnit: CompilationUnit) : CompilationUnit = - withOptionsHelper [ "--optimize-" ] "withNoOptimize is only supported for F#" cUnit + withOptionsHelper [ "--optimize+" ] "withNoOptimize is only supported for F#" cUnit let withOptimize (cUnit: CompilationUnit) : CompilationUnit = withOptionsHelper [ "--optimize+" ] "withOptimize is only supported for F#" cUnit diff --git a/tests/FSharp.Test.Utilities/TestFramework.fs b/tests/FSharp.Test.Utilities/TestFramework.fs index 78f2ea81934..0468d3ee661 100644 --- a/tests/FSharp.Test.Utilities/TestFramework.fs +++ b/tests/FSharp.Test.Utilities/TestFramework.fs @@ -437,7 +437,7 @@ let envVars () = let initializeSuite () = #if DEBUG - let configurationName = "Debug" + let configurationName = "Release"// TODO "Debug" #else let configurationName = "Release" #endif diff --git a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs new file mode 100644 index 00000000000..cc1a6b5b189 --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs @@ -0,0 +1,231 @@ +module internal ParallelTypeCheckingTests.Code.GraphBasedOpt + +#nowarn "1182" + +open System.Collections.Generic +open System.IO +open FSharp.Compiler.Optimizer +open FSharp.Compiler.Service.Driver.OptimizeTypes +open FSharp.Compiler.TypedTreeOps +open ParallelTypeCheckingTests.Utils +open ParallelTypeCheckingTests +open FSharp.Compiler.TypedTree +open Internal.Utilities.Library +open Internal.Utilities.Library.Extras + +let collectResults (inputs: CollectorInputs) : CollectorOutputs = + let files = + inputs + |> Array.map (fun {Phase1 = phase1; Phase2 = _phase2; Phase3 = phase3} -> + let (_, _, implFileOptData, _), optimizeDuringCodeGen = phase1 + let _, implFile = phase3 + let implFile = + { + ImplFile = implFile + OptimizeDuringCodeGen = optimizeDuringCodeGen + } + implFile, implFileOptData + ) + + let lastFilePhase1Env = + inputs + |> Array.last + |> fun {Phase1 = phase1} -> + let (optEnvPhase1, _, _, _), _ = phase1 + optEnvPhase1 + + files, lastFilePhase1Env.Full + +type FilePhaseFuncs = Phase1Fun * Phase2Fun * Phase3Fun +type FileResults = + { + mutable Phase1: Phase1Res option + mutable Phase2: Phase2Res option + mutable Phase3: Phase3Res option + } + with + member this.HasResult (phase: Phase) = + match phase with + | Phase.Phase1 -> this.Phase1 |> Option.isSome + | Phase.Phase2 -> this.Phase2 |> Option.isSome + | Phase.Phase3 -> this.Phase3 |> Option.isSome + member x.Get1 () = x.Phase1 |> Option.get + member x.Get2 () = x.Phase2 |> Option.get + member x.Get3 () = x.Phase3 |> Option.get + + static member Empty = + { + Phase1 = None + Phase2 = None + Phase3 = None + } + +module FileResults = + let complete (results: FileResults) = + let {FileResults.Phase1 = phase1; Phase2 = phase2; Phase3 = phase3} = results + match phase1, phase2, phase3 with + | Some phase1, Some phase2, Some phase3 -> {FileResultsComplete.Phase1 = phase1; Phase2 = phase2; Phase3 = phase3} + | _ -> failwith $"Unexpected lack of results" + +type WorkItem = + | Phase1 of Phase1Inputs + | Phase2 of Phase2Inputs + | Phase3 of Phase3Inputs + +type Node = + { + Phase: Phase + Idx: FileIdx + } + with override this.ToString() = $"[{this.Idx}-{this.Phase}]" +module Node = + let make phase idx = { Idx = idx; Phase = phase } + +let getPhase1Res (p: FileResults) = + p.Phase1 + |> Option.get + |> fun ((env, file, _, hidden), _) -> env, file, hidden + +let getPhase2Res (p: FileResults) = + p.Phase2 + |> Option.get + +let getPhase3Res (p: FileResults) = + p.Phase3 + |> Option.get + |> fun (env, _) -> env + +type IdxGraph = Graph + +type _Result = OptimizeRes + +let mergeHidingInfos (empty: SignatureHidingInfo) (infos: SignatureHidingInfo[]): SignatureHidingInfo = + infos + |> Array.fold SignatureHidingInfo.Union empty + +type Goer = IdxGraph -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs + +let goGraph (idxGraph: IReadOnlyDictionary) (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFuncs) (files: CheckedImplFile[]) : CollectorOutputs = + // Create a 3x graph by cloning each file with its deps for each phase. Add links from phase3 -> phase2 -> phase1 + let idxGraph = + idxGraph + // Temporary to workaround a bug in graph creation + |> Seq.map (fun (KeyValue(f, deps)) -> + f, deps |> Array.filter (fun d -> d <> f) + ) + |> readOnlyDict + let graph = + idxGraph + |> Seq.collect (fun (KeyValue(file, deps)) -> + let deps = + deps + // Temporary to workaround a bug in graph creation + |> Array.filter (fun d -> d <> file) + |> Array.map FileIdx + let file = FileIdx file + // Create a node per each phase + Phase.all + |> Array.map (fun phase -> + let cur = Node.make phase file + let deps = + deps + |> Array.map (Node.make phase) + let prevNode = + Phase.prev phase + |> Option.map (fun prev -> Node.make prev file) + let deps = + match prevNode with + | Some prev -> Array.append deps [|prev|] + | None -> deps + cur, deps + ) + ) + |> readOnlyDict + // + // graph + // |> Seq.iter (fun (KeyValue(f, deps)) -> + // let d = System.String.Join(",", deps) + // printfn $"{f} ==> {d}" + // ) + + let mergeEnvs envs = + mergeEnvs env0 envs + + let transitiveIdxGraph = + idxGraph + |> Graph.transitiveOpt + + let results = + Array.init files.Length (fun _ -> FileResults.Empty) + let getRes idx = results[idx] + let hidingInfo0 = SignatureHidingInfo.Empty + + let work (x: Node) : unit = + let {Idx=FileIdx idx; Phase=phase} = x + let file = files[idx] + let res = getRes idx + let depResults = + transitiveIdxGraph[idx] + |> Array.map getRes + + match phase with + | Phase.Phase1 -> + // take env and hidingInfo from dependencies + let env = + depResults + |> Array.map (fun r -> + let (a,_b,_c,_d), _e = r.Get1() + a.Delta + ) + |> mergeEnvs + let hidingInfo = + depResults + |> Array.map (fun r -> + let (_a,_b,_c,d), _e = r.Get1() + d.Delta + ) + |> mergeHidingInfos hidingInfo0 + let inputs = env, hidingInfo, file + let phase1Res = phase1 inputs + res.Phase1 <- Some phase1Res + | Phase.Phase2 -> + // take env from dependencies + let env = + depResults + |> Array.map (fun r -> + let a,_b = r.Get2() + a.Delta + ) + |> mergeEnvs + // Get file and hidingInfo from phase1 of the current file + let (_optEnv, file, _, hidingInfo), _ = res.Get1() + let inputs = env, hidingInfo.Full, file + let phase2Res = phase2 inputs + res.Phase2 <- Some phase2Res + | Phase.Phase3 -> + // take env from dependencies + let env = + depResults + |> Array.map (fun r -> + let a,_b = r.Get3() + a.Delta + ) + |> mergeEnvs + // Get file and hidingInfo from phase1 of the current file + let (_optEnv, _, _, hidingInfo), _ = res.Get1() + // Get impl file from phase2 + let _, file = res.Get2() + let inputs = env, hidingInfo.Full, file + let phase3Res = phase3 inputs + res.Phase3 <- Some phase3Res + + // printfn $"{x} finished" + + GraphProcessing.processGraphSimpler + graph + work + 12 + + let completeResults = results |> Array.map FileResults.complete + let collected = collectResults completeResults + collected diff --git a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs index 387371ae3a0..7b20b7983c4 100644 --- a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs +++ b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs @@ -1,6 +1,7 @@ /// Parallel processing of graph of work items with dependencies module ParallelTypeCheckingTests.GraphProcessing +open System.Collections.Concurrent open System.Collections.Generic open System.Threading @@ -264,9 +265,6 @@ let processGraph<'Item, 'State, 'Result, 'FinalFileResult when 'Item: equality a nodesArray |> Array.filter (fun node -> includeInFinalState node.Info.Item) |> Array.sortBy (fun node -> node.Info.Item) - |> fun nodes -> - // printfn $"%+A{nodes |> Array.map (fun n -> n.Info.Item.ToString())}" - nodes |> Array.fold (fun (fileResults, state) node -> let fileResult, state = folder state (node.Result.Value |> snd) @@ -274,3 +272,186 @@ let processGraph<'Item, 'State, 'Result, 'FinalFileResult when 'Item: equality a ([||], emptyState) finals, state + +type Node2<'Item, 'Result> = + { + Info: NodeInfo<'Item> + mutable ProcessedDepsCount: int + mutable Result: 'Result option + } + +// TODO Could be replaced with a simpler recursive approach with memoised per-item results +let processGraphSimple<'Item, 'Result when 'Item: equality and 'Item: comparison> + (graph: Graph<'Item>) + // Accepts item and a list of item results. Handles combining results. + (doWork: 'Item -> ResultWrapper<'Item, 'Result>[] -> 'Result) + (parallelism: int) + : ResultWrapper<'Item, 'Result>[] = + let transitiveDeps = graph |> Graph.transitiveOpt + let dependants = graph |> Graph.reverse + + let makeNode (item: 'Item) : Node2<'Item, ResultWrapper<'Item, 'Result>> = + let info = + let exists = graph.ContainsKey item + if + not exists + || not (transitiveDeps.ContainsKey item) + || not (dependants.ContainsKey item) + then + failwith $"WHAT {item}" + + { + Item = item + Deps = graph[item] + TransitiveDeps = transitiveDeps[item] + Dependants = dependants[item] + } + + { + Info = info + Result = None + ProcessedDepsCount = 0 + } + + let nodes = graph.Keys |> Seq.map (fun item -> item, makeNode item) |> readOnlyDict + let lookup item = nodes[item] + let lookupMany items = items |> Array.map lookup + + let leaves = + nodes.Values + |> Seq.filter (fun n -> n.Info.Deps.Length = 0) + |> Seq.toArray + + printfn $"Node count: {nodes.Count}" + + let work + (node: Node2<'Item, ResultWrapper<'Item, 'Result>>) + : Node2<'Item, ResultWrapper<'Item, 'Result>>[] = + let _deps = lookupMany node.Info.Deps + let transitiveDeps = lookupMany node.Info.TransitiveDeps + let inputs = + transitiveDeps + |> Array.map (fun n -> n.Result |> Option.get) + let singleRes = doWork node.Info.Item inputs + let singleRes = + { + Item = node.Info.Item + Result = singleRes + } + node.Result <- Some singleRes + // Need to double-check that only one dependency schedules this dependant + let unblocked = + node.Info.Dependants + |> lookupMany + |> Array.filter (fun x -> + let pdc = + // TODO Not ideal, better ways most likely exist + lock x (fun () -> + x.ProcessedDepsCount <- x.ProcessedDepsCount + 1 + x.ProcessedDepsCount) + pdc = x.Info.Deps.Length + ) + unblocked + + use cts = new CancellationTokenSource() + + Parallel.processInParallel + leaves + work + parallelism + (fun processedCount -> processedCount = nodes.Count) + cts.Token + (fun x -> x.Info.Item.ToString()) + + let nodesArray = nodes.Values |> Seq.toArray + + nodesArray + |> Array.map (fun n -> n.Result.Value) + + +/// Used for processing +type NodeInfo3<'Item> = + { + Item: 'Item + Deps: 'Item[] + Dependants: 'Item[] + } + +type Node3<'Item> = + { + Info: NodeInfo3<'Item> + mutable ProcessedDepsCount: int + } + +/// Graph processing that doesn't handle results but just invokes the worker when dependencies are ready +let processGraphSimpler<'Item when 'Item: equality and 'Item: comparison> + (graph: Graph<'Item>) + // Accepts item and a list of item results. Handles combining results. + (doWork: 'Item -> unit) + (parallelism: int) + : unit + = + let dependants = graph |> Graph.reverse + + let makeNode (item: 'Item) : Node3<'Item> = + let info = + let exists = graph.ContainsKey item + if + not exists + || not (dependants.ContainsKey item) + then + failwith $"WHAT {item}" + { + Item = item + Deps = graph[item] + Dependants = dependants[item] + } + + { + Info = info + ProcessedDepsCount = 0 + } + + let nodes = graph.Keys |> Seq.map (fun item -> item, makeNode item) |> readOnlyDict + let lookup item = nodes[item] + let lookupMany items = items |> Array.map lookup + + let leaves = + nodes.Values + |> Seq.filter (fun n -> n.Info.Deps.Length = 0) + |> Seq.toArray + + // printfn $"Node count: {nodes.Count}" + + let work + (node: Node3<'Item>) + : Node3<'Item>[] + = + let _deps = lookupMany node.Info.Deps + // printfn $"{node.Info.Item} DoWork" + doWork node.Info.Item + // printfn $"{node.Info.Item} DoneWork" + // Need to double-check that only one dependency schedules this dependant + let unblocked = + node.Info.Dependants + |> lookupMany + |> Array.filter (fun x -> + let pdc = + // TODO Not ideal, better ways most likely exist + lock x (fun () -> + x.ProcessedDepsCount <- x.ProcessedDepsCount + 1 + x.ProcessedDepsCount) + pdc = x.Info.Deps.Length + ) + // printfn $"{node.Info.Item} unblocked gathered" + unblocked + + use cts = new CancellationTokenSource() + + Parallel.processInParallel + leaves + work + parallelism + (fun processedCount -> processedCount = nodes.Count) + cts.Token + (fun x -> x.Info.Item.ToString()) diff --git a/tests/ParallelTypeCheckingTests/Code/Parallel.fs b/tests/ParallelTypeCheckingTests/Code/Parallel.fs index 45e6379b8db..fa0e9b80fac 100644 --- a/tests/ParallelTypeCheckingTests/Code/Parallel.fs +++ b/tests/ParallelTypeCheckingTests/Code/Parallel.fs @@ -24,18 +24,21 @@ let processInParallel let mutable processedCount = 0 let processItem item = - printfn $"Processing {_itemToString item}" + // printfn $"{_itemToString item} Processing" let toSchedule = work item + // printfn $"{_itemToString item} worked" let processedCount = lock processedCountLock (fun () -> processedCount <- processedCount + 1 processedCount) + + // printfn $"{_itemToString item} after lock" - let toScheduleString = - toSchedule |> Array.map _itemToString |> (fun names -> String.Join(", ", names)) + // let toScheduleString = + // toSchedule |> Array.map _itemToString |> (fun names -> String.Join(", ", names)) - printfn $"Scheduling {toSchedule.Length} items: {toScheduleString}" + // printfn $"{_itemToString item} finished - scheduling {toSchedule.Length} items: {toScheduleString}" toSchedule |> Array.iter bc.Add processedCount @@ -50,3 +53,4 @@ let processInParallel // TODO Do we need to handle cancellation given that workers do it already? Array.Parallel.map workerWork (Array.init parallelism (fun _ -> ())) |> ignore + diff --git a/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs b/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs index b4a037d84ed..064069977cc 100644 --- a/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs +++ b/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs @@ -14,6 +14,8 @@ open FSharp.Compiler.CompilerConfig open FSharp.Compiler.CompilerImports open FSharp.Compiler.DiagnosticsLogger open FSharp.Compiler.NameResolution +open FSharp.Compiler.OptimizeInputs +open FSharp.Compiler.Optimizer open FSharp.Compiler.ParseAndCheckInputs open ParallelTypeCheckingTests.FileInfoGathering open ParallelTypeCheckingTests.Types @@ -57,8 +59,43 @@ let CheckMultipleInputsInParallel |> List.map (fun ast -> ast.FileName, ast) |> readOnlyDict |> ConcurrentDictionary<_, _> - + + let fileMap = + sourceFiles + |> Array.map (fun f -> f.AST.FileName, f.Idx.Idx) + |> Map.ofArray + + let convertDep (dep: int) = + match sourceFiles[dep].AST with + | AST.ImplFile _ -> dep + | AST.SigFile _ -> + let sigFile = sourceFiles[dep].AST.FileName + let implFile = sigFile.TrimEnd('i') + fileMap[implFile] + let graph = DepResolving.DependencyResolution.detectFileDependencies sourceFiles + ParseAndCheckInputs.graph <- + let implIndices = + inputs + |> List.mapi (fun i x -> i, x) + |> List.choose (fun (i, ast) -> + match ast with + | AST.SigFile _ -> None + | AST.ImplFile _ -> Some i + ) + |> List.mapi (fun i idx -> idx, i) + |> readOnlyDict + + let implGraph = + graph.Graph + |> Seq.map (fun (KeyValue(f, deps)) -> f, deps) + |> Seq.map (fun (f, deps) -> f.Idx.Idx, deps |> Array.map (fun f -> convertDep f.Idx.Idx)) + |> Seq.filter (fun (f, _) -> implIndices.ContainsKey f) + |> Seq.map (fun (f, deps) -> + f |> fun x -> implIndices[x], deps |> Array.filter implIndices.ContainsKey |> Array.map (fun x -> implIndices[x]) + ) + |> readOnlyDict + implGraph let mutable nextIdx = (graph.Files |> Array.map (fun f -> f.File.Idx.Idx) |> Array.max) + 1 @@ -259,6 +296,6 @@ let CheckMultipleInputsInParallel (fun file -> file.Idx.Idx) state (fun it -> not <| it.Name.EndsWith(".fsix")) - 10 + 12 partialResults |> Array.toList, tcState) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs new file mode 100644 index 00000000000..2178acc8e5a --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs @@ -0,0 +1,73 @@ +module ParallelTypeCheckingTests.Code.TrieApproach.AutoOpenDetection + +open FSharp.Compiler.Syntax + +let private autoOpenShapes = + set + [| + "FSharp.Core.AutoOpenAttribute" + "Core.AutoOpenAttribute" + "AutoOpenAttribute" + "FSharp.Core.AutoOpen" + "Core.AutoOpen" + "AutoOpen" + |] + +/// This isn't bullet proof but I wonder who would really alias this very core attribute. +let isAutoOpenAttribute (attribute: SynAttribute) = + match attribute.ArgExpr with + | SynExpr.Const(constant = SynConst.Unit) + | SynExpr.Const(constant = SynConst.Unit) + | SynExpr.Const(constant = SynConst.String _) + | SynExpr.Paren(expr = SynExpr.Const(constant = SynConst.String _)) -> + let attributeName = + attribute.TypeName.LongIdent + |> List.map (fun ident -> ident.idText) + |> String.concat "." + + autoOpenShapes.Contains attributeName + | _ -> false + +let isAnyAttributeAutoOpen (attributes: SynAttributes) = + List.exists (fun (atl: SynAttributeList) -> List.exists isAutoOpenAttribute atl.Attributes) attributes + +let rec hasNestedModuleWithAutoOpenAttribute (decls: SynModuleDecl list) : bool = + decls + |> List.exists (function + | SynModuleDecl.NestedModule (moduleInfo = SynComponentInfo (attributes = attributes); decls = decls) -> + isAnyAttributeAutoOpen attributes || hasNestedModuleWithAutoOpenAttribute decls + | _ -> false) + +let rec hasNestedSigModuleWithAutoOpenAttribute (decls: SynModuleSigDecl list) : bool = + decls + |> List.exists (function + | SynModuleSigDecl.NestedModule (moduleInfo = SynComponentInfo (attributes = attributes); moduleDecls = decls) -> + isAnyAttributeAutoOpen attributes + || hasNestedSigModuleWithAutoOpenAttribute decls + | _ -> false) + +let hasAutoOpenAttributeInFile (ast: ParsedInput) : bool = + match ast with + | ParsedInput.SigFile (ParsedSigFileInput (contents = contents)) -> + contents + |> List.exists (fun (SynModuleOrNamespaceSig (attribs = attribs; decls = decls)) -> + isAnyAttributeAutoOpen attribs || hasNestedSigModuleWithAutoOpenAttribute decls) + | ParsedInput.ImplFile (ParsedImplFileInput (contents = contents)) -> + contents + |> List.exists (fun (SynModuleOrNamespace (attribs = attribs; decls = decls)) -> + isAnyAttributeAutoOpen attribs || hasNestedModuleWithAutoOpenAttribute decls) + +// ============================================================================================================================== +// ============================================================================================================================== + +open System.IO +open NUnit.Framework +open FSharp.Compiler.Service.Tests.Common + +[] +let ``detect auto open`` () = + let file = + Path.Combine(__SOURCE_DIRECTORY__, "..", "..", "..", "..", "src", "Compiler", "Utilities", "ImmutableArray.fsi") + + let ast = parseSourceCode (file, File.ReadAllText(file)) + Assert.True(hasAutoOpenAttributeInFile ast) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs new file mode 100644 index 00000000000..378a702cb31 --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs @@ -0,0 +1,560 @@ +module ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution + +open System.Linq +open FSharp.Compiler.Syntax +open Internal.Utilities.Library.Extras + +// This is pseudo code of how we could restructure the trie code +// My main benefit is that you can easily visually inspect if an identifier will match something in the trie + +// This code just looks for a path in the trie +// It could be cached and is easy to reason about. +let queryTrie (trie: TrieNode) (path: ModuleSegment list) : QueryTrieNodeResult = + let rec visit (currentNode: TrieNode) (path: ModuleSegment list) = + match path with + | [] -> failwith "path should not be empty" + | [ lastNodeFromPath ] -> + let childResults = + currentNode.Children + |> Seq.tryFind (fun (KeyValue (segment, _childNode)) -> segment = lastNodeFromPath) + + match childResults with + | None -> QueryTrieNodeResult.NodeDoesNotExist + | Some (KeyValue (_, childNode)) -> + if Set.isEmpty childNode.Files then + QueryTrieNodeResult.NodeDoesNotExposeData + else + QueryTrieNodeResult.NodeExposesData(childNode.Files) + | currentPath :: restPath -> + let childResults = + currentNode.Children + |> Seq.tryFind (fun (KeyValue (segment, _childNode)) -> segment = currentPath) + + match childResults with + | None -> QueryTrieNodeResult.NodeDoesNotExist + | Some (KeyValue (_, childNode)) -> visit childNode restPath + + visit trie path + +let queryTrieMemoized (trie: TrieNode) : QueryTrie = + Internal.Utilities.Library.Tables.memoize (queryTrie trie) + +// Now how to detect the deps between files? +// Process the content of each file using some state + +// Helper function to process a open statement +// The statement could link to files and/or should be tracked as an open namespace +let processOpenPath (queryTrie: QueryTrie) (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = + let queryResult = queryTrie path + + match queryResult with + | QueryTrieNodeResult.NodeDoesNotExist -> state + | QueryTrieNodeResult.NodeDoesNotExposeData -> state.AddOpenNamespace path + | QueryTrieNodeResult.NodeExposesData files -> state.AddDependenciesAndOpenNamespace(files, path) + +// Helper function to process an identifier +let processIdentifier (queryTrie: QueryTrie) (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = + let queryResult = queryTrie path + + match queryResult with + | QueryTrieNodeResult.NodeDoesNotExist -> state + | QueryTrieNodeResult.NodeDoesNotExposeData -> + // This can occur when you are have a file that uses a known namespace (for example namespace System). + // When any other code uses that System namespace it won't find anything in the user code. + state + | QueryTrieNodeResult.NodeExposesData files -> state.AddDependencies files + +// Typically used to folder FileContentEntry items over a FileContentQueryState +let rec processStateEntry (queryTrie: QueryTrie) (state: FileContentQueryState) (entry: FileContentEntry) : FileContentQueryState = + match entry with + | FileContentEntry.TopLevelNamespace (topLevelPath, content) -> + let state = + match topLevelPath with + | [] -> state + | _ -> processOpenPath queryTrie topLevelPath state + + List.fold (processStateEntry queryTrie) state content + + | FileContentEntry.OpenStatement path -> + // An open statement can directly reference file or be a partial open statement + // Both cases need to be processed. + let stateAfterFullOpenPath = processOpenPath queryTrie path state + + // Any existing open statement could be extended with the current path (if that node where to exists in the trie) + // The extended path could add a new link (in case of a module or namespace with types) + // It might also not add anything at all (in case it the extended path is still a partial one) + (stateAfterFullOpenPath, state.OpenNamespaces) + ||> Seq.fold (fun acc openNS -> processOpenPath queryTrie [ yield! openNS; yield! path ] acc) + + | FileContentEntry.PrefixedIdentifier path -> + match path with + | [] -> + // should not be possible though + state + | _ -> + // path could consist out of multiple segments + (state, [| 1 .. path.Length |]) + ||> Seq.fold (fun state takeParts -> + let path = List.take takeParts path + // process the name was if it were a FQN + let stateAfterFullIdentifier = processIdentifier queryTrie path state + + // Process the name in combination with the existing open namespaces + (stateAfterFullIdentifier, state.OpenNamespaces) + ||> Seq.fold (fun acc openNS -> processIdentifier queryTrie [ yield! openNS; yield! path ] acc)) + + | FileContentEntry.NestedModule (nestedContent = nestedContent) -> + // We don't want our current state to be affect by any open statements in the nested module + let nestedState = List.fold (processStateEntry queryTrie) state nestedContent + // Afterward we are only interested in the found dependencies in the nested module + let foundDependencies = + Set.union state.FoundDependencies nestedState.FoundDependencies + + { state with + FoundDependencies = foundDependencies + } + +let getFileNameBefore (files: FileWithAST array) idx = + files.[0 .. (idx - 1)] |> Array.map (fun f -> f.Idx) |> Set.ofArray + +let time msg f a = + let sw = System.Diagnostics.Stopwatch.StartNew() + let result = f a + sw.Stop() + printfn $"{msg} took %A{sw.Elapsed}" + result + +let mkGraph (files: FileWithAST array) = + // Implementation files backed by signatures should be excluded to construct the trie. + let trieInput = + files + |> Array.filter (fun f -> + match f.AST with + | ParsedInput.SigFile _ -> true + | ParsedInput.ImplFile _ -> Array.forall (fun (sigFile: FileWithAST) -> sigFile.File <> $"{f.File}i") files) + + let trie = time "TrieMapping.mkTrie" TrieMapping.mkTrie trieInput + + let queryTrie: QueryTrie = queryTrieMemoized trie + + let fileContents = + time "FileContentMapping.mkFileContent" Array.Parallel.map FileContentMapping.mkFileContent files + + let filesWithAutoOpen = + trieInput + |> Array.filter (fun f -> AutoOpenDetection.hasAutoOpenAttributeInFile f.AST) + |> Array.map (fun f -> f.Idx) + + time + "mkGraph" + Array.Parallel.map + (fun (file: FileWithAST) -> + let fileContent = fileContents.[file.Idx] + let knownFiles = getFileNameBefore files file.Idx + + // Process all entries of a file and query the trie when required to find the dependent files. + let result = + Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create file.Idx knownFiles) fileContent + + let allDependencies = + if filesWithAutoOpen.Length > 0 then + // Automatically add all files that came before the current file that use the [] attribute. + let autoOpenDependencies = + set ([| 0 .. (file.Idx - 1) |].Intersect(filesWithAutoOpen)) + + Set.union result.FoundDependencies autoOpenDependencies + else + result.FoundDependencies + + file, Set.toArray allDependencies) + files + +// ============================================================================================================= +// ============================================================================================================= + +open NUnit.Framework +open FSharp.Compiler.Service.Tests.Common + +let mkGraphAndReport files = + let filesWithAST = + files + |> Array.Parallel.mapi (fun idx file -> + { + Idx = idx + AST = parseSourceCode (file, System.IO.File.ReadAllText(file)) + File = file + }) + + let graph = mkGraph filesWithAST + + for fileName, deps in graph do + let depString = + deps + |> Array.map (fun depIdx -> filesWithAST.[depIdx].File) + |> String.concat "\n " + + if deps.Length = 0 then + printfn $"%s{fileName.File}: []" + else + printfn $"%s{fileName.File}:\n {depString}" + +[] +let ``Fantomas.Core for realzies`` () = + [| + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\Fantomas.Core.AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\ISourceTextExtensions.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\RangeHelpers.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\TriviaTypes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Utils.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceParser.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Version.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Queue.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\FormatConfig.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceTransformer.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Context.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Validation.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fs" + |] + |> mkGraphAndReport + +let fcsFiles = + [| + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSComp.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSIstrings.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\UtilsStrings.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\buildproperties.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.InternalsVisibleTo.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.AssemblyInfo.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\sformat.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\sformat.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\sr.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\sr.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ResizeArray.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ResizeArray.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\HashMultiMap.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\HashMultiMap.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\EditDistance.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\EditDistance.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\TaggedCollections.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\TaggedCollections.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\illib.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\illib.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\FileSystem.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\FileSystem.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ildiag.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ildiag.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\zmap.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\zmap.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\zset.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\zset.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\XmlAdapters.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\XmlAdapters.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\InternalCollections.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\InternalCollections.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\QueueList.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\QueueList.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\lib.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\lib.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ImmutableArray.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ImmutableArray.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\rational.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\rational.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\PathMap.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\PathMap.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\RidHelpers.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\range.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\range.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\Logger.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\Logger.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\LanguageFeatures.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\LanguageFeatures.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticOptions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticOptions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\TextLayoutRender.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\TextLayoutRender.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticsLogger.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticsLogger.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticResolutionHints.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticResolutionHints.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\prim-lexing.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\prim-lexing.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\prim-parsing.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\prim-parsing.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\ReferenceResolver.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\ReferenceResolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\SimulatedMSBuildReferenceResolver.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\SimulatedMSBuildReferenceResolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\CompilerLocation.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\CompilerLocation.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\BuildGraph.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\BuildGraph.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\il.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\il.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilx.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilx.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilascii.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilascii.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\ilpars.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\illex.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilprint.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilprint.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilmorph.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilmorph.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilsign.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilsign.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilnativeres.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilnativeres.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilsupp.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilsupp.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilbinary.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilbinary.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilread.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilread.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilwritepdb.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilwritepdb.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilwrite.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilwrite.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilreflect.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilreflect.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\PrettyNaming.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\PrettyNaming.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\UnicodeLexing.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\UnicodeLexing.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\XmlDoc.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\XmlDoc.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTrivia.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTrivia.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTree.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTree.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTreeOps.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTreeOps.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\ParseHelpers.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\ParseHelpers.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pppars.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pars.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\LexHelpers.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\LexHelpers.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pplex.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\\lex.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\LexFilter.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\LexFilter.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\tainted.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\tainted.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypeProviders.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypeProviders.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\QuotationPickler.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\QuotationPickler.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\CompilerGlobalState.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\CompilerGlobalState.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTree.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTree.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreeBasics.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreeBasics.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TcGlobals.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreeOps.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreeOps.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreePickle.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreePickle.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\import.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\import.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\TypeHierarchy.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\TypeHierarchy.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\infos.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\infos.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AccessibilityLogic.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AccessibilityLogic.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AttributeChecking.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AttributeChecking.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\TypeRelations.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\TypeRelations.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\InfoReader.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\InfoReader.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\NicePrint.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\NicePrint.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AugmentWithHashCompare.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AugmentWithHashCompare.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\NameResolution.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\NameResolution.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\SignatureConformance.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\SignatureConformance.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\MethodOverrides.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\MethodOverrides.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\MethodCalls.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\MethodCalls.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\PatternMatchCompilation.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\PatternMatchCompilation.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\ConstraintSolver.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\ConstraintSolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckFormatStrings.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckFormatStrings.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\FindUnsolved.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\FindUnsolved.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\QuotationTranslator.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\QuotationTranslator.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\PostInferenceChecks.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\PostInferenceChecks.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckBasics.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckBasics.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckExpressions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckExpressions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckPatterns.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckPatterns.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckComputationExpressions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckComputationExpressions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckIncrementalClasses.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckIncrementalClasses.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckDeclarations.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckDeclarations.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\Optimizer.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\Optimizer.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\DetupleArgs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\DetupleArgs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\InnerLambdasToTopLevelFuncs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\InnerLambdasToTopLevelFuncs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerCalls.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerCalls.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerSequences.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerSequences.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerComputedCollections.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerComputedCollections.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerStateMachines.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerStateMachines.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerLocalMutables.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerLocalMutables.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\EraseClosures.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\EraseClosures.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\EraseUnions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\EraseUnions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\IlxGen.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\IlxGen.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\FxResolver.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\FxResolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/AssemblyResolveHandler.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/AssemblyResolveHandler.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/NativeDllResolveHandler.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/NativeDllResolveHandler.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/DependencyProvider.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/DependencyProvider.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerConfig.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerConfig.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerImports.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerImports.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerDiagnostics.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerDiagnostics.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\ParseAndCheckInputs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\ParseAndCheckInputs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\ScriptClosure.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\ScriptClosure.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerOptions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerOptions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\OptimizeInputs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\OptimizeInputs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\XmlDocFileWriter.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\XmlDocFileWriter.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\BinaryResourceFormats.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\BinaryResourceFormats.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\StaticLinking.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\StaticLinking.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CreateILModule.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CreateILModule.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\fsc.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\fsc.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\FSharpDiagnostic.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\FSharpDiagnostic.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\SymbolHelpers.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\SymbolHelpers.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\Symbols.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\Symbols.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\Exprs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\Exprs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\SymbolPatterns.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\SymbolPatterns.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\SemanticClassification.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\SemanticClassification.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ItemKey.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ItemKey.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\SemanticClassificationKey.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\SemanticClassificationKey.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpSource.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpSource.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\IncrementalBuild.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\IncrementalBuild.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceCompilerDiagnostics.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceCompilerDiagnostics.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceConstants.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceDeclarationLists.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceDeclarationLists.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceLexing.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceLexing.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParseTreeWalk.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParseTreeWalk.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceNavigation.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceNavigation.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParamInfoLocations.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParamInfoLocations.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpParseFileResults.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpParseFileResults.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParsedInputOps.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParsedInputOps.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceAssemblyContent.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceAssemblyContent.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceXmlDocParser.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceXmlDocParser.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ExternalSymbol.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ExternalSymbol.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\QuickParse.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\QuickParse.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpCheckerResults.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpCheckerResults.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\service.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\service.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceInterfaceStubGenerator.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceInterfaceStubGenerator.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceStructure.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceStructure.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceAnalysis.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceAnalysis.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Interactive\ControlledExecution.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Interactive\fsi.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Interactive\fsi.fs" + // @"C:\projekty\fsharp\fsharp_main\src\Compiler\Legacy\LegacyMSBuildReferenceResolver.fsi" + // @"C:\projekty\fsharp\fsharp_main\src\Compiler\Legacy\LegacyMSBuildReferenceResolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Legacy\LegacyHostedCompilerForTesting.fs" + |] + +[] +let ``FCS for realzies`` () = mkGraphAndReport fcsFiles + +[] +let ``FCS for debugging`` () = + let filesWithAST = + fcsFiles + |> Array.mapi (fun idx file -> + { + Idx = idx + AST = parseSourceCode (file, System.IO.File.ReadAllText(file)) + File = file + }) + + let contents = + Array.map (fun (file: FileWithAST) -> FileContentMapping.mkFileContent file) filesWithAST + + ignore contents diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs new file mode 100644 index 00000000000..c2f98f9c799 --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs @@ -0,0 +1,691 @@ +module rec ParallelTypeCheckingTests.Code.TrieApproach.FileContentMapping + +open FSharp.Compiler.Syntax +open FSharp.Compiler.SyntaxTreeOps + +[] +module Continuation = + let rec sequence<'a, 'ret> (recursions: (('a -> 'ret) -> 'ret) list) (finalContinuation: 'a list -> 'ret) : 'ret = + match recursions with + | [] -> [] |> finalContinuation + | recurse :: recurses -> recurse (fun ret -> sequence recurses (fun rets -> ret :: rets |> finalContinuation)) + +type Continuations = ((FileContentEntry list -> FileContentEntry list) -> FileContentEntry list) list + +/// Option.toList >> (List.collect f) +let cfo f a = lc f (Option.toList a) +/// List.collect +let lc = List.collect + +let longIdentToPath (skipLast: bool) (longId: LongIdent) : ModuleSegment list = + if skipLast then + List.take (longId.Length - 1) longId + else + longId + |> List.map (fun ident -> ident.idText) + +let synLongIdentToPath (skipLast: bool) (synLongIdent: SynLongIdent) = + longIdentToPath skipLast synLongIdent.LongIdent + +let visitSynLongIdent (lid: SynLongIdent) : FileContentEntry list = visitLongIdent lid.LongIdent + +let visitLongIdent (lid: LongIdent) = + match lid with + | [] + | [ _ ] -> [] + | lid -> [ FileContentEntry.PrefixedIdentifier(longIdentToPath true lid) ] + +let visitSynAttribute (a: SynAttribute) : FileContentEntry list = + [ yield! visitSynLongIdent a.TypeName; yield! visitSynExpr a.ArgExpr ] + +let visitSynAttributeList (attributes: SynAttributeList) : FileContentEntry list = + lc visitSynAttribute attributes.Attributes + +let visitSynAttributes (attributes: SynAttributes) : FileContentEntry list = lc visitSynAttributeList attributes + +let visitSynModuleDecl (decl: SynModuleDecl) : FileContentEntry list = + match decl with + | SynModuleDecl.Open(target = SynOpenDeclTarget.ModuleOrNamespace (longId, _)) -> + [ FileContentEntry.OpenStatement(synLongIdentToPath false longId) ] + | SynModuleDecl.Open(target = SynOpenDeclTarget.Type (typeName, _)) -> visitSynType typeName + | SynModuleDecl.Attributes (attributes, _) -> lc visitSynAttributeList attributes + | SynModuleDecl.Expr (expr, _) -> visitSynExpr expr + | SynModuleDecl.NestedModule (moduleInfo = SynComponentInfo (longId = [ ident ]; attributes = attributes); decls = decls) -> + [ + yield! visitSynAttributes attributes + yield FileContentEntry.NestedModule(ident.idText, lc visitSynModuleDecl decls) + ] + | SynModuleDecl.NestedModule _ -> failwith "A nested module cannot have multiple identifiers" + | SynModuleDecl.Let (bindings = bindings) -> lc visitBinding bindings + | SynModuleDecl.Types (typeDefns = typeDefns) -> lc visitSynTypeDefn typeDefns + | SynModuleDecl.HashDirective _ -> [] + | SynModuleDecl.ModuleAbbrev (longId = longId) -> + // I believe this is enough + // A module abbreviation doesn't appear to be exposed as part of the current module/namespace + visitLongIdent longId + | SynModuleDecl.NamespaceFragment _ -> [] + | SynModuleDecl.Exception(exnDefn = SynExceptionDefn (exnRepr = SynExceptionDefnRepr (attributes = attributes + caseName = caseName + longId = longId) + members = members)) -> + [ + yield! visitSynAttributes attributes + yield! visitSynUnionCase caseName + yield! cfo visitLongIdent longId + yield! lc visitSynMemberDefn members + ] + +let visitSynModuleSigDecl (md: SynModuleSigDecl) = + match md with + | SynModuleSigDecl.Open(target = SynOpenDeclTarget.ModuleOrNamespace (longId, _)) -> + [ FileContentEntry.OpenStatement(synLongIdentToPath false longId) ] + | SynModuleSigDecl.Open(target = SynOpenDeclTarget.Type (typeName, _)) -> visitSynType typeName + | SynModuleSigDecl.NestedModule (moduleInfo = SynComponentInfo (longId = [ ident ]; attributes = attributes); moduleDecls = decls) -> + [ + yield! visitSynAttributes attributes + yield FileContentEntry.NestedModule(ident.idText, lc visitSynModuleSigDecl decls) + ] + | SynModuleSigDecl.NestedModule _ -> failwith "A nested module cannot have multiple identifiers" + | SynModuleSigDecl.ModuleAbbrev _ -> failwith "no support for module abbreviations" + | SynModuleSigDecl.Val (valSig, _) -> visitSynValSig valSig + | SynModuleSigDecl.Types (types = types) -> lc visitSynTypeDefnSig types + | SynModuleSigDecl.Exception(exnSig = SynExceptionSig (exnRepr = SynExceptionDefnRepr (attributes = attributes + caseName = caseName + longId = longId) + members = members)) -> + [ + yield! visitSynAttributes attributes + yield! visitSynUnionCase caseName + yield! cfo visitLongIdent longId + yield! lc visitSynMemberSig members + ] + | SynModuleSigDecl.HashDirective _ -> [] + | SynModuleSigDecl.NamespaceFragment _ -> [] + +let visitSynUnionCase (SynUnionCase (attributes = attributes; caseType = caseType)) = + let caseEntries = + match caseType with + | SynUnionCaseKind.Fields cases -> lc visitSynField cases + | SynUnionCaseKind.FullType (fullType = fullType) -> visitSynType fullType + + [ yield! visitSynAttributes attributes; yield! caseEntries ] + +let visitSynEnumCase (SynEnumCase (attributes = attributes)) = visitSynAttributes attributes + +let visitSynTypeDefn + (SynTypeDefn (typeInfo = SynComponentInfo (attributes = attributes; typeParams = typeParams; constraints = constraints) + typeRepr = typeRepr + members = members)) + : FileContentEntry list = + let reprEntries = + match typeRepr with + | SynTypeDefnRepr.Simple (simpleRepr, _) -> + match simpleRepr with + | SynTypeDefnSimpleRepr.Union (unionCases = unionCases) -> lc visitSynUnionCase unionCases + | SynTypeDefnSimpleRepr.Enum (cases = cases) -> lc visitSynEnumCase cases + | SynTypeDefnSimpleRepr.Record (recordFields = recordFields) -> lc visitSynField recordFields + // This is only used in the typed tree + // The parser doesn't construct this + | SynTypeDefnSimpleRepr.General _ -> [] + | SynTypeDefnSimpleRepr.LibraryOnlyILAssembly _ -> [] + | SynTypeDefnSimpleRepr.TypeAbbrev (rhsType = rhsType) -> visitSynType rhsType + | SynTypeDefnSimpleRepr.None _ -> [] + // This is only used in the typed tree + // The parser doesn't construct this + | SynTypeDefnSimpleRepr.Exception _ -> [] + | SynTypeDefnRepr.ObjectModel (kind, members, _) -> + match kind with + | SynTypeDefnKind.Delegate (signature, _) -> [ yield! visitSynType signature; yield! lc visitSynMemberDefn members ] + | _ -> lc visitSynMemberDefn members + | SynTypeDefnRepr.Exception _ -> + // This is only used in the typed tree + // The parser doesn't construct this + [] + + [ + yield! visitSynAttributes attributes + yield! cfo visitSynTyparDecls typeParams + yield! lc visitSynTypeConstraint constraints + yield! reprEntries + yield! lc visitSynMemberDefn members + ] + +let visitSynTypeDefnSig + (SynTypeDefnSig (typeInfo = SynComponentInfo (attributes = attributes; typeParams = typeParams; constraints = constraints) + typeRepr = typeRepr + members = members)) + = + let reprEntries = + match typeRepr with + | SynTypeDefnSigRepr.Simple (simpleRepr, _) -> + match simpleRepr with + | SynTypeDefnSimpleRepr.Union (unionCases = unionCases) -> lc visitSynUnionCase unionCases + | SynTypeDefnSimpleRepr.Enum (cases = cases) -> lc visitSynEnumCase cases + | SynTypeDefnSimpleRepr.Record (recordFields = recordFields) -> lc visitSynField recordFields + // This is only used in the typed tree + // The parser doesn't construct this + | SynTypeDefnSimpleRepr.General _ -> [] + | SynTypeDefnSimpleRepr.LibraryOnlyILAssembly _ -> [] + | SynTypeDefnSimpleRepr.TypeAbbrev (rhsType = rhsType) -> visitSynType rhsType + | SynTypeDefnSimpleRepr.None _ -> [] + // This is only used in the typed tree + // The parser doesn't construct this + | SynTypeDefnSimpleRepr.Exception _ -> [] + | SynTypeDefnSigRepr.ObjectModel (kind, members, _) -> + match kind with + | SynTypeDefnKind.Delegate (signature, _) -> [ yield! visitSynType signature; yield! lc visitSynMemberSig members ] + | _ -> lc visitSynMemberSig members + | SynTypeDefnSigRepr.Exception _ -> + // This is only used in the typed tree + // The parser doesn't construct this + [] + + [ + yield! visitSynAttributes attributes + yield! cfo visitSynTyparDecls typeParams + yield! lc visitSynTypeConstraint constraints + yield! reprEntries + yield! lc visitSynMemberSig members + ] + +let visitSynValSig (SynValSig (attributes = attributes; synType = synType; synExpr = synExpr)) = + [ + yield! visitSynAttributes attributes + yield! visitSynType synType + yield! cfo visitSynExpr synExpr + ] + +let visitSynField (SynField (attributes = attributes; fieldType = fieldType)) = + [ yield! visitSynAttributes attributes; yield! visitSynType fieldType ] + +let visitSynMemberDefn (md: SynMemberDefn) : FileContentEntry list = + match md with + | SynMemberDefn.Member (memberDefn = binding) -> visitBinding binding + | SynMemberDefn.Open _ -> [] + | SynMemberDefn.GetSetMember (memberDefnForGet, memberDefnForSet, _, _) -> + [ + yield! cfo visitBinding memberDefnForGet + yield! cfo visitBinding memberDefnForSet + ] + | SynMemberDefn.ImplicitCtor (ctorArgs = ctorArgs) -> visitSynSimplePats ctorArgs + | SynMemberDefn.ImplicitInherit (inheritType, inheritArgs, _, _) -> [ yield! visitSynType inheritType; yield! visitSynExpr inheritArgs ] + | SynMemberDefn.LetBindings (bindings = bindings) -> lc visitBinding bindings + | SynMemberDefn.AbstractSlot (slotSig = slotSig) -> visitSynValSig slotSig + | SynMemberDefn.Interface (interfaceType, _, members, _) -> + [ + yield! visitSynType interfaceType + yield! cfo (lc visitSynMemberDefn) members + ] + | SynMemberDefn.Inherit (baseType, _, _) -> visitSynType baseType + | SynMemberDefn.ValField (fieldInfo, _) -> visitSynField fieldInfo + | SynMemberDefn.NestedType _ -> [] + | SynMemberDefn.AutoProperty (attributes = attributes; typeOpt = typeOpt; synExpr = synExpr) -> + [ + yield! visitSynAttributes attributes + yield! cfo visitSynType typeOpt + yield! visitSynExpr synExpr + ] + +let visitSynInterfaceImpl (SynInterfaceImpl (interfaceTy = t; bindings = bindings; members = members)) = + [ + yield! visitSynType t + yield! lc visitBinding bindings + yield! lc visitSynMemberDefn members + ] + +let visitSynType (t: SynType) : FileContentEntry list = + let rec visit (t: SynType) (continuation: FileContentEntry list -> FileContentEntry list) = + match t with + | SynType.LongIdent lid -> continuation (visitSynLongIdent lid) + | SynType.App (typeName = typeName; typeArgs = typeArgs) -> + let continuations = List.map visit (typeName :: typeArgs) + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.LongIdentApp (typeName = typeName; longDotId = longDotId; typeArgs = typeArgs) -> + let continuations = List.map visit (typeName :: typeArgs) + + let finalContinuation nodes = + visitSynLongIdent longDotId @ lc id nodes |> continuation + + Continuation.sequence continuations finalContinuation + | SynType.Tuple (path = path) -> + let continuations = List.map visit (getTypeFromTuplePath path) + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.AnonRecd (fields = fields) -> + let continuations = List.map (snd >> visit) fields + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.Array (elementType = elementType) -> visit elementType continuation + | SynType.Fun (argType, returnType, _, _) -> + let continuations = List.map visit [ argType; returnType ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.Var _ -> continuation [] + | SynType.Anon _ -> continuation [] + | SynType.WithGlobalConstraints (typeName, constraints, _) -> + visit typeName (fun nodes -> nodes @ lc visitSynTypeConstraint constraints |> continuation) + | SynType.HashConstraint (innerType, _) -> visit innerType continuation + | SynType.MeasureDivide (dividend, divisor, _) -> + let continuations = List.map visit [ dividend; divisor ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.MeasurePower (baseMeasure = baseMeasure) -> visit baseMeasure continuation + | SynType.StaticConstant _ -> continuation [] + | SynType.StaticConstantExpr (expr, _) -> continuation (visitSynExpr expr) + | SynType.StaticConstantNamed (ident, value, _) -> + let continuations = List.map visit [ ident; value ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.Paren (innerType, _) -> visit innerType continuation + | SynType.SignatureParameter (attributes = attributes; usedType = usedType) -> + visit usedType (fun nodes -> [ yield! visitSynAttributes attributes; yield! nodes ] |> continuation) + | SynType.Or (lhsType, rhsType, _, _) -> + let continuations = List.map visit [ lhsType; rhsType ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + + visit t id + +let visitSynValTyparDecls (SynValTyparDecls (typars = typars)) = cfo visitSynTyparDecls typars + +let visitSynTyparDecls (td: SynTyparDecls) : FileContentEntry list = + match td with + | SynTyparDecls.PostfixList (decls, constraints, _) -> + [ + yield! lc visitSynTyparDecl decls + yield! lc visitSynTypeConstraint constraints + ] + | SynTyparDecls.PrefixList (decls = decls) -> lc visitSynTyparDecl decls + | SynTyparDecls.SinglePrefix (decl = decl) -> visitSynTyparDecl decl + +let visitSynTyparDecl (SynTyparDecl (attributes = attributes)) = visitSynAttributes attributes + +let visitSynTypeConstraint (tc: SynTypeConstraint) : FileContentEntry list = + match tc with + | SynTypeConstraint.WhereSelfConstrained _ -> [] + | SynTypeConstraint.WhereTyparIsValueType _ -> [] + | SynTypeConstraint.WhereTyparIsReferenceType _ -> [] + | SynTypeConstraint.WhereTyparIsUnmanaged _ -> [] + | SynTypeConstraint.WhereTyparSupportsNull _ -> [] + | SynTypeConstraint.WhereTyparIsComparable _ -> [] + | SynTypeConstraint.WhereTyparIsEquatable _ -> [] + | SynTypeConstraint.WhereTyparDefaultsToType (typeName = typeName) -> visitSynType typeName + | SynTypeConstraint.WhereTyparSubtypeOfType (typeName = typeName) -> visitSynType typeName + | SynTypeConstraint.WhereTyparSupportsMember (typars, memberSig, _) -> + [ yield! visitSynType typars; yield! visitSynMemberSig memberSig ] + | SynTypeConstraint.WhereTyparIsEnum (typeArgs = typeArgs) -> lc visitSynType typeArgs + | SynTypeConstraint.WhereTyparIsDelegate (typeArgs = typeArgs) -> lc visitSynType typeArgs + +let visitSynExpr (e: SynExpr) : FileContentEntry list = + let rec visit (e: SynExpr) (continuation: FileContentEntry list -> FileContentEntry list) : FileContentEntry list = + match e with + | SynExpr.Const _ -> continuation [] + | SynExpr.Paren (expr = expr) -> visit expr continuation + | SynExpr.Quote (operator = operator; quotedExpr = quotedExpr) -> + visit operator (fun operatorNodes -> visit quotedExpr (fun quotedNodes -> operatorNodes @ quotedNodes |> continuation)) + | SynExpr.Typed (expr, targetType, _) -> visit expr (fun nodes -> nodes @ visitSynType targetType |> continuation) + | SynExpr.Tuple (exprs = exprs) -> + let continuations = List.map visit exprs + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.AnonRecd (copyInfo = copyInfo; recordFields = recordFields) -> + let continuations = + match copyInfo with + | None -> List.map (fun (_, _, e) -> visit e) recordFields + | Some (cp, _) -> visit cp :: List.map (fun (_, _, e) -> visit e) recordFields + + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.ArrayOrList (exprs = exprs) -> + let continuations = List.map visit exprs + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.Record (baseInfo = baseInfo; copyInfo = copyInfo; recordFields = recordFields) -> + let fieldNodes = + recordFields + |> lc (fun (SynExprRecordField (fieldName = (si, _); expr = expr)) -> + [ yield! visitSynLongIdent si; yield! cfo visitSynExpr expr ]) + + match baseInfo, copyInfo with + | Some (t, e, _, _, _), None -> + visit e (fun nodes -> [ yield! visitSynType t; yield! nodes; yield! fieldNodes ] |> continuation) + | None, Some (e, _) -> visit e (fun nodes -> nodes @ fieldNodes |> continuation) + | _ -> continuation fieldNodes + | SynExpr.New (targetType = targetType; expr = expr) -> visit expr (fun nodes -> visitSynType targetType @ nodes |> continuation) + | SynExpr.ObjExpr (objType, argOptions, _, bindings, members, extraImpls, _, _) -> + [ + yield! visitSynType objType + yield! cfo (fst >> visitSynExpr) argOptions + yield! lc visitBinding bindings + yield! lc visitSynMemberDefn members + yield! lc visitSynInterfaceImpl extraImpls + ] + |> continuation + | SynExpr.While (whileExpr = whileExpr; doExpr = doExpr) -> + visit whileExpr (fun whileNodes -> visit doExpr (fun doNodes -> whileNodes @ doNodes |> continuation)) + | SynExpr.For (identBody = identBody; toBody = toBody; doBody = doBody) -> + let continuations = List.map visit [ identBody; toBody; doBody ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.ForEach (pat = pat; enumExpr = enumExpr; bodyExpr = bodyExpr) -> + visit enumExpr (fun enumNodes -> + visit bodyExpr (fun bodyNodes -> [ yield! visitPat pat; yield! enumNodes; yield! bodyNodes ] |> continuation)) + | SynExpr.ArrayOrListComputed (expr = expr) -> visit expr continuation + | SynExpr.IndexRange (expr1 = expr1; expr2 = expr2) -> + match expr1, expr2 with + | None, None -> continuation [] + | Some e, None + | None, Some e -> visit e continuation + | Some e1, Some e2 -> visit e1 (fun e1Nodes -> visit e2 (fun e2Nodes -> e1Nodes @ e2Nodes |> continuation)) + | SynExpr.IndexFromEnd (expr, _) -> visit expr continuation + | SynExpr.ComputationExpr (expr = expr) -> visit expr continuation + | SynExpr.Lambda (args = args; body = body) -> visit body (fun bodyNodes -> visitSynSimplePats args @ bodyNodes |> continuation) + | SynExpr.MatchLambda (matchClauses = clauses) -> lc visitSynMatchClause clauses |> continuation + | SynExpr.Match (expr = expr; clauses = clauses) -> + visit expr (fun exprNodes -> [ yield! exprNodes; yield! lc visitSynMatchClause clauses ] |> continuation) + | SynExpr.Do (expr, _) -> visit expr continuation + | SynExpr.Assert (expr, _) -> visit expr continuation + | SynExpr.App (funcExpr = funcExpr; argExpr = argExpr) -> + visit funcExpr (fun funcNodes -> visit argExpr (fun argNodes -> funcNodes @ argNodes |> continuation)) + | SynExpr.TypeApp (expr = expr; typeArgs = typeArgs) -> + visit expr (fun exprNodes -> exprNodes @ lc visitSynType typeArgs |> continuation) + | SynExpr.LetOrUse (bindings = bindings; body = body) -> visit body (fun nodes -> lc visitBinding bindings @ nodes |> continuation) + | SynExpr.TryWith (tryExpr = tryExpr; withCases = withCases) -> + visit tryExpr (fun nodes -> nodes @ lc visitSynMatchClause withCases |> continuation) + | SynExpr.TryFinally (tryExpr = tryExpr; finallyExpr = finallyExpr) -> + visit tryExpr (fun tNodes -> visit finallyExpr (fun fNodes -> tNodes @ fNodes |> continuation)) + | SynExpr.Lazy (expr, _) -> visit expr continuation + | SynExpr.Sequential (expr1 = expr1; expr2 = expr2) -> + visit expr1 (fun nodes1 -> visit expr2 (fun nodes2 -> nodes1 @ nodes2 |> continuation)) + | SynExpr.IfThenElse (ifExpr = ifExpr; thenExpr = thenExpr; elseExpr = elseExpr) -> + let continuations = List.map visit (ifExpr :: thenExpr :: Option.toList elseExpr) + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.Typar _ -> continuation [] + | SynExpr.Ident _ -> continuation [] + | SynExpr.LongIdent (longDotId = longDotId) -> continuation (visitSynLongIdent longDotId) + | SynExpr.LongIdentSet (longDotId, expr, _) -> visit expr (fun nodes -> visitSynLongIdent longDotId @ nodes |> continuation) + | SynExpr.DotGet (expr = expr; longDotId = longDotId) -> + visit expr (fun nodes -> visitSynLongIdent longDotId @ nodes |> continuation) + | SynExpr.DotSet (targetExpr, longDotId, rhsExpr, _) -> + visit targetExpr (fun tNodes -> + visit rhsExpr (fun rNodes -> + [ yield! tNodes; yield! visitSynLongIdent longDotId; yield! rNodes ] + |> continuation)) + | SynExpr.Set (targetExpr, rhsExpr, _) -> + let continuations = List.map visit [ targetExpr; rhsExpr ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.DotIndexedGet (objectExpr, indexArgs, _, _) -> + let continuations = List.map visit [ objectExpr; indexArgs ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.DotIndexedSet (objectExpr, indexArgs, valueExpr, _, _, _) -> + let continuations = List.map visit [ objectExpr; indexArgs; valueExpr ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.NamedIndexedPropertySet (longDotId, expr1, expr2, _) -> + visit expr1 (fun nodes1 -> + visit expr2 (fun nodes2 -> + [ yield! visitSynLongIdent longDotId; yield! nodes1; yield! nodes2 ] + |> continuation)) + | SynExpr.DotNamedIndexedPropertySet (targetExpr, longDotId, argExpr, rhsExpr, _) -> + let continuations = List.map visit [ targetExpr; argExpr; rhsExpr ] + + let finalContinuation nodes = + visitSynLongIdent longDotId @ lc id nodes |> continuation + + Continuation.sequence continuations finalContinuation + | SynExpr.TypeTest (expr, targetType, _) -> visit expr (fun nodes -> nodes @ visitSynType targetType |> continuation) + | SynExpr.Upcast (expr, targetType, _) -> visit expr (fun nodes -> nodes @ visitSynType targetType |> continuation) + | SynExpr.Downcast (expr, targetType, _) -> visit expr (fun nodes -> nodes @ visitSynType targetType |> continuation) + | SynExpr.InferredUpcast (expr, _) -> visit expr continuation + | SynExpr.InferredDowncast (expr, _) -> visit expr continuation + | SynExpr.Null _ -> continuation [] + | SynExpr.AddressOf (expr = expr) -> visit expr continuation + | SynExpr.TraitCall (supportTys, traitSig, argExpr, _) -> + visit argExpr (fun nodes -> + [ + yield! visitSynType supportTys + yield! visitSynMemberSig traitSig + yield! nodes + ] + |> continuation) + | SynExpr.JoinIn (lhsExpr, _, rhsExpr, _) -> + let continuations = List.map visit [ lhsExpr; rhsExpr ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.ImplicitZero _ -> continuation [] + | SynExpr.SequentialOrImplicitYield (_, expr1, expr2, _, _) -> + let continuations = List.map visit [ expr1; expr2 ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.YieldOrReturn (expr = expr) -> visit expr continuation + | SynExpr.YieldOrReturnFrom (expr = expr) -> visit expr continuation + | SynExpr.LetOrUseBang (pat = pat; rhs = rhs; andBangs = andBangs; body = body) -> + let continuations = + let andBangExprs = List.map (fun (SynExprAndBang (body = body)) -> body) andBangs + List.map visit (body :: rhs :: andBangExprs) + + let finalContinuation nodes = + [ + yield! lc id nodes + yield! visitPat pat + yield! lc (fun (SynExprAndBang (pat = pat)) -> visitPat pat) andBangs + ] + |> continuation + + Continuation.sequence continuations finalContinuation + | SynExpr.MatchBang (expr = expr; clauses = clauses) -> + visit expr (fun exprNodes -> [ yield! exprNodes; yield! lc visitSynMatchClause clauses ] |> continuation) + | SynExpr.DoBang (expr, _) -> visit expr continuation + | SynExpr.LibraryOnlyILAssembly (typeArgs = typeArgs; args = args; retTy = retTy) -> + let typeNodes = lc visitSynType (typeArgs @ retTy) + let continuations = List.map visit args + let finalContinuation nodes = lc id nodes @ typeNodes |> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.LibraryOnlyStaticOptimization (constraints, expr, optimizedExpr, _) -> + let constraintTypes = + constraints + |> List.choose (function + | SynStaticOptimizationConstraint.WhenTyparTyconEqualsTycon (rhsType = t) -> Some t + | SynStaticOptimizationConstraint.WhenTyparIsStruct _ -> None) + + visit expr (fun eNodes -> + visit optimizedExpr (fun oNodes -> + [ yield! lc visitSynType constraintTypes; yield! eNodes; yield! oNodes ] + |> continuation)) + | SynExpr.LibraryOnlyUnionCaseFieldGet (expr, longId, _, _) -> + visit expr (fun eNodes -> visitLongIdent longId @ eNodes |> continuation) + | SynExpr.LibraryOnlyUnionCaseFieldSet (expr, longId, _, rhsExpr, _) -> + visit expr (fun eNodes -> + visit rhsExpr (fun rhsNodes -> [ yield! visitLongIdent longId; yield! eNodes; yield! rhsNodes ] |> continuation)) + | SynExpr.ArbitraryAfterError _ -> continuation [] + | SynExpr.FromParseError _ -> continuation [] + | SynExpr.DiscardAfterMissingQualificationAfterDot _ -> continuation [] + | SynExpr.Fixed (expr, _) -> visit expr continuation + | SynExpr.InterpolatedString (contents = contents) -> + let continuations = + List.map + visit + (List.choose + (function + | SynInterpolatedStringPart.FillExpr (fillExpr = e) -> Some e + | SynInterpolatedStringPart.String _ -> None) + contents) + + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.DebugPoint _ -> continuation [] + | SynExpr.Dynamic (funcExpr, _, argExpr, _) -> + let continuations = List.map visit [ funcExpr; argExpr ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + + visit e id + +let visitPat (p: SynPat) : FileContentEntry list = + let rec visit (p: SynPat) (continuation: FileContentEntry list -> FileContentEntry list) : FileContentEntry list = + match p with + | SynPat.Paren (pat = pat) -> visit pat continuation + | SynPat.Typed (pat = pat; targetType = t) -> visit pat (fun nodes -> nodes @ visitSynType t) + | SynPat.Const _ -> continuation [] + | SynPat.Wild _ -> continuation [] + | SynPat.Named _ -> continuation [] + | SynPat.Attrib (pat, attributes, _) -> visit pat (fun nodes -> visitSynAttributes attributes @ nodes |> continuation) + | SynPat.Or (lhsPat, rhsPat, _, _) -> + let continuations = List.map visit [ lhsPat; rhsPat ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.ListCons (lhsPat, rhsPat, _, _) -> + let continuations = List.map visit [ lhsPat; rhsPat ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.Ands (pats, _) -> + let continuations = List.map visit pats + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.As (lhsPat, rhsPat, _) -> + let continuations = List.map visit [ lhsPat; rhsPat ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.LongIdent (longDotId = longDotId; typarDecls = typarDecls; argPats = argPats) -> + continuation + [ + yield! visitSynLongIdent longDotId + yield! cfo visitSynValTyparDecls typarDecls + yield! visitSynArgPats argPats + ] + | SynPat.Tuple (_, elementPats, _) -> + let continuations = List.map visit elementPats + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.ArrayOrList (_, elementPats, _) -> + let continuations = List.map visit elementPats + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.Record (fieldPats, _) -> + let pats = List.map (fun (_, _, p) -> p) fieldPats + let lids = lc (fun ((l, _), _, _) -> visitLongIdent l) fieldPats + let continuations = List.map visit pats + + let finalContinuation nodes = + [ yield! lc id nodes; yield! lids ] |> continuation + + Continuation.sequence continuations finalContinuation + | SynPat.Null _ -> continuation [] + | SynPat.OptionalVal _ -> continuation [] + | SynPat.IsInst (t, _) -> continuation (visitSynType t) + | SynPat.QuoteExpr (expr, _) -> continuation (visitSynExpr expr) + | SynPat.DeprecatedCharRange _ -> continuation [] + | SynPat.InstanceMember _ -> continuation [] + | SynPat.FromParseError _ -> continuation [] + + visit p id + +let visitSynArgPats (argPat: SynArgPats) = + match argPat with + | SynArgPats.Pats args -> lc visitPat args + | SynArgPats.NamePatPairs (pats = pats) -> lc (fun (_, _, p) -> visitPat p) pats + +let visitSynSimplePat (pat: SynSimplePat) = + match pat with + | SynSimplePat.Id _ -> [] + | SynSimplePat.Attrib (pat, attributes, _) -> [ yield! visitSynSimplePat pat; yield! visitSynAttributes attributes ] + | SynSimplePat.Typed (pat, t, _) -> [ yield! visitSynSimplePat pat; yield! visitSynType t ] + +let visitSynSimplePats (pats: SynSimplePats) = + match pats with + | SynSimplePats.SimplePats (pats = pats) -> lc visitSynSimplePat pats + | SynSimplePats.Typed (pats, t, _) -> [ yield! visitSynSimplePats pats; yield! visitSynType t ] + +let visitSynMatchClause (SynMatchClause (pat = pat; whenExpr = whenExpr; resultExpr = resultExpr)) = + [ + yield! visitPat pat + yield! cfo visitSynExpr whenExpr + yield! visitSynExpr resultExpr + ] + +let visitBinding (SynBinding (attributes = attributes; headPat = headPat; returnInfo = returnInfo; expr = expr)) : FileContentEntry list = + let pattern = + match headPat with + | SynPat.LongIdent(argPats = SynArgPats.Pats pats) -> lc visitPat pats + | _ -> visitPat headPat + + [ + yield! visitSynAttributes attributes + yield! pattern + yield! cfo visitSynBindingReturnInfo returnInfo + yield! visitSynExpr expr + ] + +let visitSynBindingReturnInfo (SynBindingReturnInfo (typeName = typeName; attributes = attributes)) = + [ yield! visitSynAttributes attributes; yield! visitSynType typeName ] + +let visitSynMemberSig (ms: SynMemberSig) : FileContentEntry list = + match ms with + | SynMemberSig.Member (memberSig = memberSig) -> visitSynValSig memberSig + | SynMemberSig.Interface (interfaceType, _) -> visitSynType interfaceType + | SynMemberSig.Inherit (inheritedType, _) -> visitSynType inheritedType + | SynMemberSig.ValField (field, _) -> visitSynField field + | SynMemberSig.NestedType _ -> [] + +let mkFileContent (f: FileWithAST) : FileContentEntry list = + match f.AST with + | ParsedInput.SigFile (ParsedSigFileInput (contents = contents)) -> + lc + (fun (SynModuleOrNamespaceSig (longId = longId; kind = kind; decls = decls; attribs = attribs)) -> + let attributes = lc visitSynAttributeList attribs + + let contentEntries = + match kind with + | SynModuleOrNamespaceKind.GlobalNamespace + | SynModuleOrNamespaceKind.AnonModule -> lc visitSynModuleSigDecl decls + | SynModuleOrNamespaceKind.DeclaredNamespace -> + let path = longIdentToPath false longId + + [ FileContentEntry.TopLevelNamespace(path, lc visitSynModuleSigDecl decls) ] + | SynModuleOrNamespaceKind.NamedModule -> + let path = longIdentToPath true longId + + [ FileContentEntry.TopLevelNamespace(path, lc visitSynModuleSigDecl decls) ] + + [ yield! attributes; yield! contentEntries ]) + contents + + | ParsedInput.ImplFile (ParsedImplFileInput (contents = contents)) -> + lc + (fun (SynModuleOrNamespace (longId = longId; attribs = attribs; kind = kind; decls = decls)) -> + let attributes = lc visitSynAttributeList attribs + + let contentEntries = + match kind with + | SynModuleOrNamespaceKind.GlobalNamespace + | SynModuleOrNamespaceKind.AnonModule -> lc visitSynModuleDecl decls + | SynModuleOrNamespaceKind.DeclaredNamespace -> + let path = longIdentToPath false longId + + [ FileContentEntry.TopLevelNamespace(path, lc visitSynModuleDecl decls) ] + | SynModuleOrNamespaceKind.NamedModule -> + let path = longIdentToPath true longId + + [ FileContentEntry.TopLevelNamespace(path, lc visitSynModuleDecl decls) ] + + [ yield! attributes; yield! contentEntries ]) + contents + +// ================================================================================================================================ +// ================================================================================================================================ +module Tests = + open NUnit.Framework + open FSharp.Compiler.Service.Tests.Common + + [] + let ``Test a single file`` () = + let fileName = + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fs" + + let ast = parseSourceCode (fileName, System.IO.File.ReadAllText(fileName)) + let contents = mkFileContent { Idx = 0; File = fileName; AST = ast } + ignore contents diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs new file mode 100644 index 00000000000..0b105a3cdf8 --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs @@ -0,0 +1,800 @@ +module ParallelTypeCheckingTests.Code.TrieApproach.SampleData + +open System.Collections.Generic +open NUnit.Framework +open ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution + +// This file contains some hard coded data to easily debug the various aspects of the dependency resolution. + +// Some helper DSL functions to construct the FileContentEntry items +// This should again be mapped from the AST + +let topLevelNS (topLevelNamespaceString: string) (content: FileContentEntry list) = + topLevelNamespaceString.Split(".") + |> Array.toList + |> fun name -> FileContentEntry.TopLevelNamespace(name, content) + +let topLevelMod (topLevelModuleString: string) (content: FileContentEntry list) = + let parts = topLevelModuleString.Split(".") + + parts + |> Array.take (parts.Length - 1) + |> Array.toList + |> fun name -> FileContentEntry.TopLevelNamespace(name, content) + +let openSt (openStatement: string) = + openStatement.Split(".") |> Array.toList |> FileContentEntry.OpenStatement + +let nestedModule name content = + FileContentEntry.NestedModule(name, content) + +let prefIdent (lid: string) = + let parts = lid.Split(".") + Array.take (parts.Length - 1) parts |> List.ofArray |> PrefixedIdentifier + +// Some hardcoded files that reflect the file content of the first files in the Fantomas.Core project. +let files = + [| + { + Name = "AssemblyInfo.fs" + Idx = 0 + Content = + [| + topLevelNS + "System" + [ + openSt "System.Runtime.CompilerServices" + nestedModule "AssemblyVersionInformation" [] + ] + |] + } + { + Name = "ISourceTextExtensions.fs" + Idx = 1 + Content = + [| + topLevelMod + "Fantomas.Core.ISourceTextExtensions" + [ + openSt "System.Text" + openSt "FSharp.Compiler.Text" + prefIdent "range.StartLine" + prefIdent "this.GetLineString" + prefIdent "range.StartLine" + prefIdent "range.EndLine" + prefIdent "range.EndColumn" + prefIdent "range.StartColumn" + prefIdent "line.Substring" + prefIdent "sb.AppendLine" + prefIdent "lastLine.Substring" + ] + |] + } + { + Name = "RangeHelpers.fs" + Idx = 2 + Content = + [| + topLevelNS + "Fantomas.Core" + [ + openSt "FSharp.Compiler.Text" + nestedModule + "RangeHelpers" + [ + prefIdent "Position.posGeq" + prefIdent "b.Start" + prefIdent "a.Start" + prefIdent "a.End" + prefIdent "b.End" + prefIdent "Range.equals" + prefIdent "r1.FileName" + prefIdent "r2.FileName" + prefIdent "r1.End" + prefIdent "r2.Start" + prefIdent "r1.EndColumn" + prefIdent "r2.StartColumn" + prefIdent "Range.mkRange" + prefIdent "r.FileName" + prefIdent "r.Start" + prefIdent "Position.mkPos" + prefIdent "r.StartLine" + prefIdent "r.StartColumn" + prefIdent "r.EndLine" + prefIdent "r.EndColumn" + prefIdent "r.End" + prefIdent "List.sortBy" + prefIdent "List.reduce" + prefIdent "Range.unionRanges" + ] + nestedModule + "RangePatterns" + [ + prefIdent "RangeHelpers.mkStartEndRange" + prefIdent "range.FileName" + prefIdent "range.Start" + prefIdent "range.StartLine" + prefIdent "range.StartColumn" + ] + ] + |] + } + { + Name = "AstExtensions.fsi" + Idx = 3 + Content = + [| + topLevelMod "Fantomas.Core.AstExtensions" [ openSt "FSharp.Compiler.Text"; openSt "FSharp.Compiler.Syntax" ] + |] + } + { + Name = "AstExtensions.fs" + Idx = 4 + Content = + [| + topLevelMod + "Fantomas.Core.AstExtensions" + [ + openSt "FSharp.Compiler.SyntaxTrivia" + openSt "FSharp.Compiler.Text" + openSt "FSharp.Compiler.Text.Range" + openSt "FSharp.Compiler.Syntax" + prefIdent "range.Zero" + prefIdent "h.idRange" + prefIdent "List.last" + prefIdent "ident.idRange" + prefIdent "IdentTrivia.OriginalNotationWithParen" + prefIdent "IdentTrivia.HasParenthesis" + prefIdent "IdentTrivia.OriginalNotation" + prefIdent "Range.Zero" + prefIdent "single.FullRange" + prefIdent "List.fold" + prefIdent "head.FullRange" + prefIdent "fieldName.FullRange" + prefIdent "expr.Range" + prefIdent "SynModuleOrNamespaceKind.AnonModule" + prefIdent "List.tryHead" + prefIdent "List.tryLast" + prefIdent "d.Range" + prefIdent "s.Range" + prefIdent "e.Range" + prefIdent "this.Range" + prefIdent "CommentTrivia.LineComment" + prefIdent "CommentTrivia.BlockComment" + prefIdent "ConditionalDirectiveTrivia.If" + prefIdent "ConditionalDirectiveTrivia.Else" + prefIdent "ConditionalDirectiveTrivia.EndIf" + prefIdent "List.map" + prefIdent "c.Range" + prefIdent "acc.StartLine" + prefIdent "triviaRange.StartLine" + prefIdent "acc.EndLine" + prefIdent "triviaRange.EndLine" + prefIdent "ParsedInput.ImplFile" + prefIdent "r.Start" + prefIdent "m.FullRange.Start" + prefIdent "Range.Zero.Start" + prefIdent "Range.Zero.End" + prefIdent "r.End" + prefIdent "lastModule.FullRange.End" + prefIdent "this.Range.FileName" + prefIdent "trivia.CodeComments" + prefIdent "trivia.ConditionalDirectives" + prefIdent "ParsedInput.SigFile" + prefIdent "SynInterpolatedStringPart.String" + prefIdent "SynInterpolatedStringPart.FillExpr" + prefIdent "i.idRange" + prefIdent "std.FullRange" + prefIdent "a.Range" + prefIdent "RangeHelpers.mergeRanges" + prefIdent "synTypar.Range" + prefIdent "sf.FullRange" + prefIdent "head.Range" + prefIdent "b.FullRange" + prefIdent "xmlDoc.IsEmpty" + prefIdent "xmlDoc.Range" + prefIdent "attributes.IsEmpty" + prefIdent "attributes.Head.Range" + prefIdent "trivia.LeadingKeyword" + prefIdent "SynLeadingKeyword.Member" + prefIdent "SynPat.LongIdent" + prefIdent "pat.Range" + prefIdent "trivia.LeadingKeyword.Range" + ] + |] + } + { + Name = "TriviaTypes.fs" + Idx = 5 + Content = + [| + topLevelMod "Fantomas.Core.TriviaTypes" [ openSt "FSharp.Compiler.Text"; openSt "FSharp.Compiler.Syntax" ] + |] + } + { + Name = "Utils.fs" + Idx = 6 + Content = + [| + topLevelNS + "Fantomas.Core" + [ + openSt "System" + openSt "System.Text.RegularExpressions" + nestedModule "Char" [ prefIdent "c.ToString" ] + nestedModule + "String" + [ + prefIdent "str.Replace" + prefIdent "str.StartsWith" + prefIdent "StringComparison.Ordinal" + prefIdent "String.Empty" + prefIdent "source.Split" + prefIdent "StringSplitOptions.None" + prefIdent "Array.mapi" + prefIdent "Regex.IsMatch" + prefIdent "Array.choose" + prefIdent "Array.toList" + prefIdent "List.tryHead" + prefIdent "List.map" + prefIdent "String.concat" + prefIdent "List.zip" + prefIdent "String.length" + prefIdent "String.IsNullOrEmpty" + prefIdent "String.IsNullOrWhiteSpace" + prefIdent "String.exists" + ] + nestedModule + "Cache" + [ + prefIdent "System.Collections.Generic.HashSet" + prefIdent "HashIdentity.Reference" + prefIdent "cache.Contains" + prefIdent "cache.Add" + prefIdent "System.Collections.Concurrent.ConcurrentDictionary" + prefIdent "HashIdentity.Structural" + prefIdent "cache.GetOrAdd" + prefIdent "this.Equals" + prefIdent "Object.ReferenceEquals" + prefIdent "this.GetHashCode" + ] + nestedModule + "Dict" + [ + prefIdent "System.Collections.Generic.IDictionary" + prefIdent "d.TryGetValue" + ] + nestedModule + "List" + [ + prefIdent "List.takeWhile" + prefIdent "List.choose" + prefIdent "List.isEmpty" + prefIdent "List.rev" + ] + nestedModule "Map" [ prefIdent "Map.tryFind" ] + nestedModule "Async" [ prefIdent "async.Bind"; prefIdent "async.Return" ] + nestedModule "Continuation" [] + ] + |] + } + { + Name = "SourceParser.fs" + Idx = 7 + Content = + [| + topLevelMod + "Fantomas.Core.SourceParser" + [ + openSt "System" + openSt "FSharp.Compiler.Syntax" + openSt "FSharp.Compiler.Syntax.PrettyNaming" + openSt "FSharp.Compiler.SyntaxTrivia" + openSt "FSharp.Compiler.Text" + openSt "FSharp.Compiler.Xml" + openSt "Fantomas.Core" + openSt "Fantomas.Core.AstExtensions" + openSt "Fantomas.Core.TriviaTypes" + openSt "Fantomas.Core.RangePatterns" + prefIdent "SynTypar.SynTypar" + prefIdent "TyparStaticReq.None" + prefIdent "TyparStaticReq.HeadType" + prefIdent "SynRationalConst.Integer" + prefIdent "SynRationalConst.Rational" + prefIdent "SynRationalConst.Negate" + prefIdent "SynConst.Unit" + prefIdent "ParsedInput.ImplFile" + prefIdent "ParsedInput.SigFile" + prefIdent "ParsedImplFileInput.ParsedImplFileInput" + prefIdent "ParsedSigFileInput.ParsedSigFileInput" + prefIdent "SynModuleOrNamespace.SynModuleOrNamespace" + prefIdent "trivia.LeadingKeyword" + prefIdent "m.FullRange" + prefIdent "SynModuleOrNamespaceSig.SynModuleOrNamespaceSig" + prefIdent "a.TypeName" + prefIdent "a.ArgExpr" + prefIdent "a.Target" + prefIdent "px.ToXmlDoc" + prefIdent "xmlDoc.UnprocessedLines" + prefIdent "xmlDoc.Range" + prefIdent "SynModuleDecl.Open" + prefIdent "SynOpenDeclTarget.ModuleOrNamespace" + prefIdent "SynOpenDeclTarget.Type" + prefIdent "SynType.LongIdent" + prefIdent "SynModuleDecl.ModuleAbbrev" + prefIdent "SynModuleDecl.HashDirective" + prefIdent "SynModuleDecl.NamespaceFragment" + prefIdent "SynModuleDecl.Attributes" + prefIdent "SynModuleDecl.Let" + prefIdent "SynModuleDecl.Expr" + prefIdent "SynModuleDecl.Types" + prefIdent "SynModuleDecl.NestedModule" + prefIdent "trivia.ModuleKeyword" + prefIdent "trivia.EqualsRange" + prefIdent "SynModuleDecl.Exception" + prefIdent "SynModuleSigDecl.Open" + prefIdent "SynModuleSigDecl.ModuleAbbrev" + prefIdent "SynModuleSigDecl.HashDirective" + prefIdent "SynModuleSigDecl.NamespaceFragment" + prefIdent "SynModuleSigDecl.Val" + prefIdent "SynModuleSigDecl.Types" + prefIdent "SynModuleSigDecl.NestedModule" + prefIdent "SynModuleSigDecl.Exception" + prefIdent "SynExceptionDefnRepr.SynExceptionDefnRepr" + prefIdent "SynExceptionDefn.SynExceptionDefn" + prefIdent "SynExceptionSig.SynExceptionSig" + prefIdent "px.IsEmpty" + prefIdent "trivia.BarRange" + prefIdent "Range.unionRanges" + prefIdent "SynUnionCaseKind.Fields" + prefIdent "SynUnionCaseKind.FullType" + prefIdent "Option.map" + prefIdent "i.idRange" + prefIdent "t.Range" + prefIdent "SynMemberDefn.NestedType" + prefIdent "SynMemberDefn.Open" + prefIdent "SynMemberDefn.ImplicitInherit" + prefIdent "SynMemberDefn.Inherit" + prefIdent "SynMemberDefn.ValField" + prefIdent "SynMemberDefn.ImplicitCtor" + prefIdent "SynMemberDefn.Member" + prefIdent "SynMemberDefn.LetBindings" + prefIdent "SynType.Fun" + prefIdent "SynMemberKind.PropertyGet" + prefIdent "SynMemberKind.PropertySet" + prefIdent "SynMemberKind.PropertyGetSet" + prefIdent "SynMemberDefn.AbstractSlot" + prefIdent "trivia.WithKeyword" + prefIdent "mf.MemberKind" + prefIdent "SynMemberDefn.Interface" + prefIdent "SynMemberDefn.AutoProperty" + prefIdent "SynMemberDefn.GetSetMember" + prefIdent "SynPat.LongIdent" + prefIdent "Position.posLt" + prefIdent "getKeyword.Start" + prefIdent "setKeyword.Start" + prefIdent "SynMemberKind.ClassConstructor" + prefIdent "SynMemberKind.Constructor" + prefIdent "SynMemberKind.Member" + prefIdent "mf.IsInstance" + prefIdent "mf.IsOverrideOrExplicitImpl" + prefIdent "SynExpr.Typed" + prefIdent "RangeHelpers.rangeEq" + prefIdent "t1.Range" + prefIdent "t2.Range" + prefIdent "Option.bind" + prefIdent "trivia.ColonRange" + prefIdent "b.FullRange" + prefIdent "SynBindingKind.Do" + prefIdent "SynLeadingKeyword.Extern" + prefIdent "SynExpr.TraitCall" + prefIdent "SynExpr.Quote" + prefIdent "SynExpr.Paren" + prefIdent "SynExpr.Lazy" + prefIdent "SynExpr.InferredDowncast" + prefIdent "SynExpr.InferredUpcast" + prefIdent "SynExpr.Assert" + prefIdent "SynExpr.AddressOf" + prefIdent "SynExpr.YieldOrReturn" + prefIdent "SynExpr.YieldOrReturnFrom" + prefIdent "SynExpr.Do" + prefIdent "SynExpr.DoBang" + prefIdent "SynExpr.Fixed" + prefIdent "SynExpr.TypeTest" + prefIdent "SynExpr.Downcast" + prefIdent "SynExpr.Upcast" + prefIdent "SynExpr.While" + prefIdent "SynExpr.For" + prefIdent "SynExpr.Null" + prefIdent "SynExpr.Const" + prefIdent "SynExpr.TypeApp" + prefIdent "SynExpr.Match" + prefIdent "trivia.MatchKeyword" + prefIdent "SynExpr.MatchBang" + prefIdent "trivia.MatchBangKeyword" + prefIdent "SynExpr.Sequential" + prefIdent "SynExpr.Ident" + prefIdent "SynExpr.LongIdent" + prefIdent "SynExpr.ComputationExpr" + prefIdent "SynExpr.App" + prefIdent "ExprAtomicFlag.NonAtomic" + prefIdent "compExpr.Range" + prefIdent "SynExpr.ArrayOrListComputed" + prefIdent "RangeHelpers.mkStartEndRange" + prefIdent "SynExpr.ArrayOrList" + prefIdent "SynExpr.Tuple" + prefIdent "SynExpr.InterpolatedString" + prefIdent "SynExpr.IndexRange" + prefIdent "SynExpr.IndexFromEnd" + prefIdent "SynExpr.Typar" + prefIdent "SynConst.Double" + prefIdent "SynConst.Decimal" + prefIdent "SynConst.Single" + prefIdent "SynConst.Int16" + prefIdent "SynConst.Int32" + prefIdent "SynConst.Int64" + prefIdent "List.moreThanOne" + prefIdent "SynExpr.Dynamic" + prefIdent "IdentTrivia.OriginalNotationWithParen" + prefIdent "originalNotation.Length" + prefIdent "originalNotation.StartsWith" + prefIdent "List.rev" + prefIdent "SynExpr.DotGet" + prefIdent "SynExpr.Lambda" + prefIdent "SynExpr.MatchLambda" + prefIdent "SynExpr.New" + prefIdent "IdentTrivia.OriginalNotation" + prefIdent "ident.idText" + prefIdent "newLineInfixOps.Contains" + prefIdent "List.length" + prefIdent "SynExpr.JoinIn" + prefIdent "SynExpr.LetOrUse" + prefIdent "xs.Length" + prefIdent "List.mapi" + prefIdent "trivia.InKeyword" + prefIdent "List.map" + prefIdent "SynExpr.LetOrUseBang" + prefIdent "List.collect" + prefIdent "Continuation.sequence" + prefIdent "SynExpr.ForEach" + prefIdent "SynExpr.DotIndexedSet" + prefIdent "SynExpr.NamedIndexedPropertySet" + prefIdent "SynExpr.DotNamedIndexedPropertySet" + prefIdent "SynExpr.DotIndexedGet" + prefIdent "SynExpr.DotSet" + prefIdent "SynExpr.IfThenElse" + prefIdent "trivia.IfKeyword" + prefIdent "trivia.IsElif" + prefIdent "trivia.ThenKeyword" + prefIdent "trivia.ElseKeyword" + prefIdent "unitRange.StartColumn" + prefIdent "unitRange.EndColumn" + prefIdent "SynExpr.Record" + prefIdent "SynExpr.AnonRecd" + prefIdent "SynExpr.ObjExpr" + prefIdent "SynExpr.LongIdentSet" + prefIdent "SynExpr.TryWith" + prefIdent "trivia.TryKeyword" + prefIdent "SynExpr.TryFinally" + prefIdent "trivia.FinallyKeyword" + prefIdent "SynExpr.ArbitraryAfterError" + prefIdent "SynExpr.FromParseError" + prefIdent "SynExpr.DiscardAfterMissingQualificationAfterDot" + prefIdent "SynExpr.LibraryOnlyILAssembly" + prefIdent "SynExpr.LibraryOnlyStaticOptimization" + prefIdent "FSharp.Core" + prefIdent "SynExpr.LibraryOnlyUnionCaseFieldGet" + prefIdent "SynExpr.LibraryOnlyUnionCaseFieldSet" + prefIdent "SynPat.OptionalVal" + prefIdent "SynPat.Attrib" + prefIdent "SynPat.Or" + prefIdent "p.Range" + prefIdent "SynPat.Ands" + prefIdent "SynPat.Null" + prefIdent "SynPat.Wild" + prefIdent "SynPat.Tuple" + prefIdent "SynPat.ArrayOrList" + prefIdent "SynPat.Typed" + prefIdent "SynPat.Named" + prefIdent "SynPat.As" + prefIdent "SynArgPats.NamePatPairs" + prefIdent "SynArgPats.Pats" + prefIdent "SynPat.ListCons" + prefIdent "trivia.ColonColonRange" + prefIdent "synLongIdent.IdentsWithTrivia" + prefIdent "synIdent.FullRange" + prefIdent "synLongIdent.FullRange" + prefIdent "SynPat.Paren" + prefIdent "SynPat.Record" + prefIdent "SynPat.Const" + prefIdent "SynPat.IsInst" + prefIdent "SynPat.QuoteExpr" + prefIdent "newIdent.idText" + prefIdent "pat.Range" + prefIdent "SynSimplePats.SimplePats" + prefIdent "SynSimplePats.Typed" + prefIdent "SynSimplePat.Attrib" + prefIdent "SynSimplePat.Id" + prefIdent "SynSimplePat.Typed" + prefIdent "trivia.ArrowRange" + prefIdent "SynMatchClause.SynMatchClause" + prefIdent "matchRange.Start" + prefIdent "clause.Range.Start" + prefIdent "me.Range" + prefIdent "SynTypeDefnSimpleRepr.Enum" + prefIdent "SynTypeDefnSimpleRepr.Union" + prefIdent "SynTypeDefnSimpleRepr.Record" + prefIdent "SynTypeDefnSimpleRepr.None" + prefIdent "SynTypeDefnSimpleRepr.TypeAbbrev" + prefIdent "SynTypeDefnSimpleRepr.General" + prefIdent "SynTypeDefnSimpleRepr.LibraryOnlyILAssembly" + prefIdent "SynTypeDefnSimpleRepr.Exception" + prefIdent "SynTypeDefnRepr.Simple" + prefIdent "SynTypeDefnRepr.ObjectModel" + prefIdent "SynTypeDefnRepr.Exception" + prefIdent "List.tryFind" + prefIdent "List.filter" + prefIdent "SynTypeDefnSigRepr.Simple" + prefIdent "SynTypeDefnSigRepr.ObjectModel" + prefIdent "SynTypeDefnSigRepr.Exception" + prefIdent "SynTypeDefnKind.Unspecified" + prefIdent "SynTypeDefnKind.Class" + prefIdent "SynTypeDefnKind.Interface" + prefIdent "SynTypeDefnKind.Struct" + prefIdent "SynTypeDefnKind.Record" + prefIdent "SynTypeDefnKind.Union" + prefIdent "SynTypeDefnKind.Abbrev" + prefIdent "SynTypeDefnKind.Opaque" + prefIdent "SynTypeDefnKind.Augmentation" + prefIdent "SynTypeDefnKind.IL" + prefIdent "SynTypeDefnKind.Delegate" + prefIdent "std.FullRange" + prefIdent "SynTyparDecls.PostfixList" + prefIdent "SynType.HashConstraint" + prefIdent "SynType.MeasurePower" + prefIdent "SynType.MeasureDivide" + prefIdent "SynType.StaticConstant" + prefIdent "SynType.StaticConstantExpr" + prefIdent "SynType.StaticConstantNamed" + prefIdent "SynType.Array" + prefIdent "SynType.Anon" + prefIdent "SynType.Var" + prefIdent "SynType.App" + prefIdent "SynType.LongIdentApp" + prefIdent "SynType.Tuple" + prefIdent "SynType.WithGlobalConstraints" + prefIdent "SynType.AnonRecd" + prefIdent "SynType.Paren" + prefIdent "SynType.SignatureParameter" + prefIdent "SynType.Or" + prefIdent "trivia.OrKeyword" + prefIdent "lid.idText" + prefIdent "x.ToString" + prefIdent "SynTypeConstraint.WhereTyparIsValueType" + prefIdent "SynTypeConstraint.WhereTyparIsReferenceType" + prefIdent "SynTypeConstraint.WhereTyparIsUnmanaged" + prefIdent "SynTypeConstraint.WhereTyparSupportsNull" + prefIdent "SynTypeConstraint.WhereTyparIsComparable" + prefIdent "SynTypeConstraint.WhereTyparIsEquatable" + prefIdent "SynTypeConstraint.WhereTyparDefaultsToType" + prefIdent "SynTypeConstraint.WhereTyparSubtypeOfType" + prefIdent "SynTypeConstraint.WhereTyparSupportsMember" + prefIdent "SynTypeConstraint.WhereTyparIsEnum" + prefIdent "SynTypeConstraint.WhereTyparIsDelegate" + prefIdent "SynTypeConstraint.WhereSelfConstrained" + prefIdent "SynMemberSig.Member" + prefIdent "SynMemberSig.Interface" + prefIdent "SynMemberSig.Inherit" + prefIdent "SynMemberSig.ValField" + prefIdent "SynMemberSig.NestedType" + prefIdent "ident.idRange" + prefIdent "e.Range" + prefIdent "List.tryLast" + prefIdent "IdentTrivia.HasParenthesis" + prefIdent "lp.idText" + prefIdent "Seq.tryHead" + prefIdent "Char.IsUpper" + prefIdent "Option.defaultValue" + prefIdent "ExprAtomicFlag.Atomic" + prefIdent "RangeHelpers.isAdjacentTo" + prefIdent "identifierExpr.Range" + prefIdent "argExpr.Range" + prefIdent "Seq.toList" + prefIdent "Seq.singleton" + prefIdent "List.exists" + ] + |] + } + |] + +let dictionary<'key, 'value when 'key: equality> (entries: ('key * 'value) seq) = + entries |> Seq.map (fun (k, v) -> KeyValuePair(k, v)) |> Dictionary + +let noChildren = Dictionary(0) +let emptyHS () = HashSet(0) + +let indexOf name = + Array.find (fun (fc: FileContent) -> fc.Name = name) files |> fun fc -> fc.Idx + +// This should be constructed from the AST, again a hard coded subset of Fantomas.Core +let fantomasCoreTrie: TrieNode = + { + Current = TrieNodeInfo.Root + Children = + dictionary + [| + "System", + { + Current = TrieNodeInfo.Namespace("System", emptyHS ()) + Children = + dictionary + [| + "AssemblyVersionInformation", + { + Current = TrieNodeInfo.Module("AssemblyVersionInformation", indexOf "AssemblyInfo.fs") + Children = noChildren + } + |] + } + "Fantomas", + { + Current = TrieNodeInfo.Namespace("Fantomas", emptyHS ()) + Children = + dictionary + [| + "Core", + { + Current = TrieNodeInfo.Namespace("Core", emptyHS ()) + Children = + dictionary + [| + "ISourceTextExtensions", + { + Current = + TrieNodeInfo.Module("ISourceTextExtensions", indexOf "ISourceTextExtensions.fs") + Children = noChildren + } + "RangeHelpers", + { + Current = TrieNodeInfo.Module("RangeHelpers", indexOf "RangeHelpers.fs") + Children = noChildren + } + "RangePatterns", + { + Current = TrieNodeInfo.Module("RangePatterns", indexOf "RangeHelpers.fs") + Children = noChildren + } + "AstExtensions", + { + Current = TrieNodeInfo.Module("AstExtensions", indexOf "AstExtensions.fsi") + Children = noChildren + } + "TriviaTypes", + { + Current = TrieNodeInfo.Module("TriviaTypes", indexOf "TriviaTypes.fs") + Children = noChildren + } + "Char", + { + Current = TrieNodeInfo.Module("Char", indexOf "Utils.fs") + Children = noChildren + } + "String", + { + Current = TrieNodeInfo.Module("String", indexOf "Utils.fs") + Children = noChildren + } + "Cache", + { + Current = TrieNodeInfo.Module("Cache", indexOf "Utils.fs") + Children = noChildren + } + "Dict", + { + Current = TrieNodeInfo.Module("Dict", indexOf "Utils.fs") + Children = noChildren + } + "List", + { + Current = TrieNodeInfo.Module("List", indexOf "Utils.fs") + Children = noChildren + } + "Map", + { + Current = TrieNodeInfo.Module("Map", indexOf "Utils.fs") + Children = noChildren + } + "Async", + { + Current = TrieNodeInfo.Module("Async", indexOf "Utils.fs") + Children = noChildren + } + "Continuation", + { + Current = TrieNodeInfo.Module("Continuation", indexOf "Utils.fs") + Children = noChildren + } + "SourceParser", + { + Current = TrieNodeInfo.Module("SourceParser", indexOf "SourceParser.fs") + Children = noChildren + } + |] + } + |] + } + |] + } + +[] +let ``Full project simulation`` () = + let graph = + files + |> Array.map (fun fileContent -> + let knownFiles = + files.[0 .. (fileContent.Idx - 1)] |> Array.map (fun f -> f.Idx) |> set + + let queryTrie: QueryTrie = queryTrieMemoized fantomasCoreTrie + + let result = + Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create fileContent.Idx knownFiles) fileContent.Content + + fileContent.Name, Set.toArray result.FoundDependencies) + + for fileName, deps in graph do + let depString = + deps |> Array.map (fun depIdx -> files.[depIdx].Name) |> String.concat ", " + + printfn $"%s{fileName}: [{depString}]" + +[] +let ``Query non existing node in trie`` () = + let result = + queryTrie fantomasCoreTrie [ "System"; "System"; "Runtime"; "CompilerServices" ] + + match result with + | QueryTrieNodeResult.NodeDoesNotExist -> Assert.Pass() + | result -> Assert.Fail $"Unexpected result: %A{result}" + +[] +let ``Query node that does not expose data in trie`` () = + let result = queryTrie fantomasCoreTrie [ "Fantomas"; "Core" ] + + match result with + | QueryTrieNodeResult.NodeDoesNotExposeData -> Assert.Pass() + | result -> Assert.Fail $"Unexpected result: %A{result}" + +[] +let ``Query module node that exposes one file`` () = + let result = + queryTrie fantomasCoreTrie [ "Fantomas"; "Core"; "ISourceTextExtensions" ] + + match result with + | QueryTrieNodeResult.NodeExposesData file -> + let file = Seq.exactlyOne file + Assert.AreEqual(indexOf "ISourceTextExtensions.fs", file) + | result -> Assert.Fail $"Unexpected result: %A{result}" + +[] +let ``ProcessOpenStatement full path match`` () = + let sourceParser = + Array.find (fun (f: FileContent) -> f.Name = "SourceParser.fs") files + + let state = + FileContentQueryState.Create + sourceParser.Idx + (set + [| + indexOf "AssemblyInfo.fs" + indexOf "ISourceTextExtensions.fs" + indexOf "RangeHelpers.fs" + indexOf "AstExtensions.fsi" + indexOf "TriviaTypes.fs" + indexOf "Utils.fs" + |]) + + let result = + processOpenPath (queryTrie fantomasCoreTrie) [ "Fantomas"; "Core"; "AstExtensions" ] state + + let dep = Seq.exactlyOne result.FoundDependencies + Assert.AreEqual(indexOf "AstExtensions.fsi", dep) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs new file mode 100644 index 00000000000..2fd8c8367aa --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs @@ -0,0 +1,254 @@ +module ParallelTypeCheckingTests.Code.TrieApproach.TrieMapping + +open System.Collections.Generic +open FSharp.Compiler.Syntax +open Microsoft.FSharp.Collections + +let mergeTrieNodes (defaultChildSize: int) (tries: TrieNode seq) = + let rec mergeTrieNodesAux (root: TrieNode) (KeyValue (k, v)) = + if root.Children.ContainsKey k then + let node = root.Children.[k] + + match node.Current, v.Current with + | TrieNodeInfo.Namespace (filesThatExposeTypes = currentFiles), TrieNodeInfo.Namespace (filesThatExposeTypes = otherFiles) -> + for otherFile in otherFiles do + do () + currentFiles.Add(otherFile) |> ignore + | _ -> () + + for kv in v.Children do + mergeTrieNodesAux node kv + + else + root.Children.Add(k, v) + + match Seq.tryExactlyOne tries with + | Some singleTrie -> + assert (singleTrie.Current = TrieNodeInfo.Root) + singleTrie + | None -> + let root = + { + Current = TrieNodeInfo.Root + Children = Dictionary<_, _>(defaultChildSize) + } + + do () + + for trie in tries do + assert (trie.Current = TrieNodeInfo.Root) + + for kv in trie.Children do + mergeTrieNodesAux root kv + + root + +let hs f = HashSet(Seq.singleton f) +let emptyHS () = HashSet(0) + +let rec mkTrieNodeFor (file: FileWithAST) : TrieNode = + match file.AST with + | ParsedInput.SigFile (ParsedSigFileInput (contents = contents)) -> + contents + |> List.choose (fun (SynModuleOrNamespaceSig (longId = longId; kind = kind; decls = decls; accessibility = _accessibility)) -> + let hasTypes = + List.exists + (function + | SynModuleSigDecl.Types _ -> true + | _ -> false) + decls + + let isNamespace = + match kind with + | SynModuleOrNamespaceKind.AnonModule + | SynModuleOrNamespaceKind.NamedModule -> false + | SynModuleOrNamespaceKind.DeclaredNamespace -> true + | SynModuleOrNamespaceKind.GlobalNamespace -> failwith "Not quite sure yet how to perceive this" + + match longId with + | [] -> None + | _ -> + let rootNode = + let rec visit continuation (xs: LongIdent) = + match xs with + | [] -> failwith "should even empty" + | [ finalPart ] -> + let name = finalPart.idText + + let current = + if isNamespace then + TrieNodeInfo.Namespace(name, (if hasTypes then hs file.Idx else emptyHS ())) + else + TrieNodeInfo.Module(name, file.Idx) + + let children = List.choose (mkTrieForNestedSigModule file.Idx) decls + + continuation ( + Dictionary<_, _>( + Seq.singleton ( + KeyValuePair( + name, + { + Current = current + Children = Dictionary(children) + } + ) + ) + ) + ) + | head :: tail -> + let name = head.idText + + visit + (fun node -> + let current = TrieNodeInfo.Namespace(name, emptyHS ()) + + Dictionary<_, _>(Seq.singleton (KeyValuePair(name, { Current = current; Children = node }))) + |> continuation) + tail + + visit id longId + + Some { Current = Root; Children = rootNode }) + |> mergeTrieNodes contents.Length + | ParsedInput.ImplFile (ParsedImplFileInput (contents = contents)) -> + contents + |> List.choose (fun (SynModuleOrNamespace (longId = longId; kind = kind; decls = decls; accessibility = _accessibility)) -> + let hasTypes = + List.exists + (function + | SynModuleDecl.Types _ -> true + | _ -> false) + decls + + let isNamespace = + match kind with + | SynModuleOrNamespaceKind.AnonModule + | SynModuleOrNamespaceKind.NamedModule -> false + | SynModuleOrNamespaceKind.DeclaredNamespace -> true + | SynModuleOrNamespaceKind.GlobalNamespace -> failwith "Not quite sure yet how to perceive this" + + match longId with + | [] -> None + | _ -> + let rootNode = + let rec visit continuation (xs: LongIdent) = + match xs with + | [] -> failwith "should even empty" + | [ finalPart ] -> + let name = finalPart.idText + + let current = + if isNamespace then + TrieNodeInfo.Namespace(name, (if hasTypes then hs file.Idx else emptyHS ())) + else + TrieNodeInfo.Module(name, file.Idx) + + let children = List.choose (mkTrieForSynModuleDecl file.Idx) decls + + continuation ( + Dictionary<_, _>( + Seq.singleton ( + KeyValuePair( + name, + { + Current = current + Children = Dictionary(children) + } + ) + ) + ) + ) + | head :: tail -> + let name = head.idText + + visit + (fun node -> + let current = TrieNodeInfo.Namespace(name, emptyHS ()) + + Dictionary<_, _>(Seq.singleton (KeyValuePair(name, { Current = current; Children = node }))) + |> continuation) + tail + + visit id longId + + Some { Current = Root; Children = rootNode }) + |> mergeTrieNodes contents.Length + +and mkTrieForSynModuleDecl (fileIndex: int) (decl: SynModuleDecl) : KeyValuePair option = + match decl with + | SynModuleDecl.NestedModule (moduleInfo = SynComponentInfo(longId = [ nestedModuleIdent ]); decls = decls) -> + let name = nestedModuleIdent.idText + let children = List.choose (mkTrieForSynModuleDecl fileIndex) decls + + Some( + KeyValuePair( + name, + { + Current = TrieNodeInfo.Module(name, fileIndex) + Children = Dictionary(children) + } + ) + ) + | _ -> None + +and mkTrieForNestedSigModule (fileIndex: int) (decl: SynModuleSigDecl) : KeyValuePair option = + match decl with + | SynModuleSigDecl.NestedModule (moduleInfo = SynComponentInfo(longId = [ nestedModuleIdent ]); moduleDecls = decls) -> + let name = nestedModuleIdent.idText + let children = List.choose (mkTrieForNestedSigModule fileIndex) decls + + Some( + KeyValuePair( + name, + { + Current = TrieNodeInfo.Module(name, fileIndex) + Children = Dictionary(children) + } + ) + ) + + | _ -> None + +let mkTrie (files: FileWithAST array) : TrieNode = + mergeTrieNodes 0 (Array.Parallel.map mkTrieNodeFor files) + +// ================================================================================================================================================== +// ================================================================================================================================================== +open FSharp.Compiler.Service.Tests.Common +open NUnit.Framework + +[] +let ``Fantomas Core trie`` () = + let files = + [| + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\Fantomas.Core.AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\ISourceTextExtensions.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\RangeHelpers.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\TriviaTypes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Utils.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceParser.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Version.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Queue.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\FormatConfig.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceTransformer.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Context.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Validation.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fsi" + |] + |> Array.mapi (fun idx file -> + let ast = parseSourceCode (file, System.IO.File.ReadAllText(file)) + { Idx = idx; File = file; AST = ast }) + + let trie = mkTrie files + ignore trie diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs new file mode 100644 index 00000000000..7f72c15b93b --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs @@ -0,0 +1,108 @@ +namespace ParallelTypeCheckingTests.Code.TrieApproach + +open System.Collections.Generic +open FSharp.Compiler.Syntax + +type File = string +type ModuleSegment = string + +type FileWithAST = + { + Idx: int + File: File + AST: ParsedInput + } + +/// There is a subtle difference a module and namespace. +/// A namespace does not necessarily expose a set of dependent files. +/// Only when the namespace exposes types that could later be inferred. +/// Children of a namespace don't automatically depend on each other for that reason +type TrieNodeInfo = + | Root + | Module of segment: string * file: int + | Namespace of segment: string * filesThatExposeTypes: HashSet + + member x.Segment = + match x with + | Root -> failwith "Root has no segment" + | Module (segment = segment) + | Namespace (segment = segment) -> segment + + member x.Files: Set = + match x with + | Root -> failwith "Root has no files" + | Module (file = file) -> Set.singleton file + | Namespace (filesThatExposeTypes = files) -> set files + +type TrieNode = + { + Current: TrieNodeInfo + Children: Dictionary + } + + member x.Files = x.Current.Files + +type FileContentEntry = + /// Any toplevel namespace a file might have. + /// In case a file has `module X.Y.Z`, then `X.Y` is considered to be the toplevel namespace + | TopLevelNamespace of path: ModuleSegment list * content: FileContentEntry list + /// The `open X.Y.Z` syntax. + | OpenStatement of path: ModuleSegment list + /// Any identifier that has more than one piece (LongIdent or SynLongIdent) in it. + /// The last part of the identifier should not be included. + | PrefixedIdentifier of path: ModuleSegment list + /// Being explicit about nested modules allows for easier reasoning what namespaces (paths) are open. + /// We can scope an `OpenStatement` to the everything that is happening inside the nested module. + | NestedModule of name: string * nestedContent: FileContentEntry list + +type FileContent = + { + Name: File + Idx: int + Content: FileContentEntry array + } + +type FileContentQueryState = + { + OpenNamespaces: Set + FoundDependencies: Set + CurrentFile: int + KnownFiles: Set + } + + static member Create (fileIndex: int) (knownFiles: Set) = + { + OpenNamespaces = Set.empty + FoundDependencies = Set.empty + CurrentFile = fileIndex + KnownFiles = knownFiles + } + + member x.AddDependencies(files: Set) : FileContentQueryState = + let files = Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies + { x with FoundDependencies = files } + + member x.AddOpenNamespace(path: ModuleSegment list) = + { x with + OpenNamespaces = Set.add path x.OpenNamespaces + } + + member x.AddDependenciesAndOpenNamespace(files: Set, path: ModuleSegment list) = + let foundDependencies = + Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies + + { x with + FoundDependencies = foundDependencies + OpenNamespaces = Set.add path x.OpenNamespaces + } + +[] +type QueryTrieNodeResult = + /// No node was found for the path in the trie + | NodeDoesNotExist + /// A node was found but it yielded no file links + | NodeDoesNotExposeData + /// A node was found with one or more file links + | NodeExposesData of Set + +type QueryTrie = ModuleSegment list -> QueryTrieNodeResult diff --git a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj index 745bef6b0c0..cb0f1defe15 100644 --- a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj +++ b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj @@ -2,7 +2,7 @@ - net7.0 + net7.0 Library false true @@ -10,7 +10,7 @@ true true false - false + true $(OtherFlags) --warnon:1182 $(NoWarn);FS0988;FS1182 $(DefineConstants);RELEASE @@ -37,6 +37,13 @@ + + + + + + + diff --git a/tests/ParallelTypeCheckingTests/Program.fs b/tests/ParallelTypeCheckingTests/Program.fs index 9e0c9dd9617..57ff4c1fa96 100644 --- a/tests/ParallelTypeCheckingTests/Program.fs +++ b/tests/ParallelTypeCheckingTests/Program.fs @@ -2,7 +2,10 @@ #nowarn "1182" +open FSharp.Compiler open FSharp.Compiler.CompilerConfig +open FSharp.Compiler.OptimizeInputs +open ParallelTypeCheckingTests.TestCompilation open ParallelTypeCheckingTests.TestUtils let _parse (argv: string[]) : Args = @@ -27,9 +30,24 @@ let _parse (argv: string[]) : Args = WorkingDir = workingDir } +open ParallelTypeCheckingTests.TestCompilationFromCmdlineArgs [] let main _argv = - let args = _parse _argv - let args = { args with LineLimit = None } - TestCompilationFromCmdlineArgs.TestCompilerFromArgs args + for _i in [1;2] do + ParseAndCheckInputs.CheckMultipleInputsUsingGraphMode <- + ParallelTypeChecking.CheckMultipleInputsInParallel + FSharp.Compiler.OptimizeInputs.goer <- ParallelTypeCheckingTests.Code.GraphBasedOpt.goGraph |> Some + let mode = + match _argv[0] with + | "graph" -> OptimizerMode.GraphBased + | "sequential" -> OptimizerMode.Sequential + | "partial" -> OptimizerMode.PartiallyParallel + | _ -> failwith $"unknown mode {_argv[0]}" + OptimizeInputs.optimizerMode <- mode + // let args = _parse _argv + // let args = { args with LineLimit = None } + let componentTests = codebases[System.Int32.Parse(_argv[1])] + let config = codebaseToConfig componentTests Method.Graph + TestCompilerFromArgs config 0 + diff --git a/tests/ParallelTypeCheckingTests/Tests/AssemblySetUp.fs b/tests/ParallelTypeCheckingTests/Tests/AssemblySetUp.fs index 351ea9d55ce..9a538b49424 100644 --- a/tests/ParallelTypeCheckingTests/Tests/AssemblySetUp.fs +++ b/tests/ParallelTypeCheckingTests/Tests/AssemblySetUp.fs @@ -3,6 +3,7 @@ open NUnit.Framework open OpenTelemetry.Trace + /// One-time setup for NUnit tests [] type AssemblySetUp() = @@ -12,7 +13,7 @@ type AssemblySetUp() = member this.SetUp() = FSharp.Compiler.ParseAndCheckInputs.CheckMultipleInputsUsingGraphMode <- ParallelTypeCheckingTests.ParallelTypeChecking.CheckMultipleInputsInParallel - + FSharp.Compiler.OptimizeInputs.goer <- ParallelTypeCheckingTests.Code.GraphBasedOpt.goGraph |> Some tracerProvider <- ParallelTypeCheckingTests.TestUtils.setupOtel () |> Some [] diff --git a/tests/ParallelTypeCheckingTests/Tests/ComponentTests.args.txt b/tests/ParallelTypeCheckingTests/Tests/ComponentTests.args.txt index 660592b844c..40e620825f2 100644 --- a/tests/ParallelTypeCheckingTests/Tests/ComponentTests.args.txt +++ b/tests/ParallelTypeCheckingTests/Tests/ComponentTests.args.txt @@ -28,7 +28,7 @@ --doc:..\..\artifacts\obj\FSharp.Compiler.ComponentTests\Debug\net7.0\FSharp.Compiler.ComponentTests.xml --keyfile:$PACKAGES$\microsoft.dotnet.arcade.sdk\8.0.0-beta.22552.1\tools\snk/MSFT.snk --publicsign+ ---optimize- +--optimize+ --tailcalls- -r:$PACKAGES$\fluentassertions\5.10.3\lib\netcoreapp2.1\FluentAssertions.dll -r:..\..\artifacts\bin\FSharp.Build\Debug\netstandard2.0\FSharp.Build.dll diff --git a/tests/ParallelTypeCheckingTests/Tests/FCS.args.txt b/tests/ParallelTypeCheckingTests/Tests/FCS.args.txt index 5c056374fb8..012de0b2cfe 100644 --- a/tests/ParallelTypeCheckingTests/Tests/FCS.args.txt +++ b/tests/ParallelTypeCheckingTests/Tests/FCS.args.txt @@ -33,7 +33,7 @@ --doc:..\..\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.xml --keyfile:$PACKAGES$\microsoft.dotnet.arcade.sdk\8.0.0-beta.22552.1\tools\snk/MSFT.snk --publicsign+ ---optimize- +--optimize+ --resource:..\..\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSComp.resources --resource:..\..\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSIstrings.resources --resource:..\..\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\UtilsStrings.resources diff --git a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs index 8045b0de5d0..2e2625e807e 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs @@ -1,9 +1,12 @@ module ParallelTypeCheckingTests.TestCompilation +open FSharp.Compiler.OptimizeInputs open FSharp.Test open FSharp.Test.Compiler open NUnit.Framework +open ParallelTypeCheckingTests.Code open ParallelTypeCheckingTests.TestUtils +open FSharp.Compiler type FProject = { @@ -118,11 +121,23 @@ let b = 1 "A.fs", """ namespace A +module A1 = + let x = 3 + type X = X of int + let y = X 5 + let foo () = 5 + let inline bar () = 7 + module B2 = + let a = "lalala" """ "B.fs", """ module B open A +let z = A.A1.x +let y = A.A1.y +let g = A.A1.x + 2 +let h = A.A1.bar() """ ] |> FProject.Make CompileOutput.Library @@ -193,6 +208,47 @@ let d (c: CType) = ] |> FProject.Make CompileOutput.Library + let fullyParallel = + [ + "A.fs", + """ +module A +let x = 1 +""" + "B.fs", + """ +module B +let x = 1 +""" + "C.fs", + """ +module C +let x = 1 +""" + "D.fs", + """ +module D +let x = 1 +""" + "E.fs", + """ +module E +let x = 1 +""" + "F.fs", + """ +module F +let x = 1 +""" + "G.fs", + """ +module G +let x = 1 +""" + ] + |> FProject.Make CompileOutput.Library + + let all = [ encodeDecodeSimple @@ -200,6 +256,7 @@ let d (c: CType) = fsFsi emptyNamespace dependentSignatures + fullyParallel ] type Case = @@ -221,7 +278,7 @@ let withMethod (method: Method) (cu: CompilationUnit) : CompilationUnit = | CompilationUnit.FS cs -> FS { cs with - Options = cs.Options @ (methodOptions method) + Options = cs.Options @ (methodOptions method) @ ["--optimize+"] } | cu -> cu @@ -240,8 +297,30 @@ let compileAValidProject (x: Case) = let codebases = Codebases.all + [] let ``Compile a valid project using graph-based type-checking`` (project: FProject) = + global.FSharp.Compiler.OptimizeInputs.optimizerMode <- OptimizerMode.Sequential + compileAValidProject + { + Method = Method.Graph + Project = project + } + +[] +let ``Compile a valid project using graph-based type-checking, parallel opt`` (project: FProject) = + global.FSharp.Compiler.OptimizeInputs.optimizerMode <- OptimizerMode.PartiallyParallel + compileAValidProject + { + Method = Method.Graph + Project = project + } + + +[] +let ``Compile a valid project using graph-based type-checking, graph opt`` (project: FProject) = + global.FSharp.Compiler.OptimizeInputs.optimizerMode <- OptimizerMode.GraphBased + OptimizeInputs.goer <- GraphBasedOpt.goGraph |> Some compileAValidProject { Method = Method.Graph diff --git a/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs b/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs index 6bc069f0fed..ca3cd135274 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs @@ -2,10 +2,12 @@ open FSharp.Compiler.CompilerConfig open FSharp.Compiler.DiagnosticsLogger +open FSharp.Compiler.OptimizeInputs open NUnit.Framework open System open FSharp.Compiler open ParallelTypeCheckingTests +open ParallelTypeCheckingTests.Code open ParallelTypeCheckingTests.TestUtils type Codebase = @@ -75,7 +77,7 @@ let internal TestCompilerFromArgs (config: Args) : unit = try let args = setupParsed config - let exit: int = CommandLineMain.mainAux (args, true, Some exiter) + let exit: int = CommandLineMain.mainAux (args, false, Some exiter) // TODO reset to true Assert.That(exit, Is.Zero) finally Environment.CurrentDirectory <- oldWorkDir @@ -91,6 +93,7 @@ let internal codebaseToConfig code method = [] [] let ``1. Test sequential type-checking`` (code: Codebase) = + OptimizeInputs.optimizerMode <- OptimizerMode.Sequential let config = codebaseToConfig code Method.Sequential TestCompilerFromArgs config @@ -98,6 +101,7 @@ let ``1. Test sequential type-checking`` (code: Codebase) = [] [] let ``2. Test parallelfs type-checking`` (code: Codebase) = + OptimizeInputs.optimizerMode <- OptimizerMode.Sequential let config = codebaseToConfig code Method.ParallelCheckingOfBackedImplFiles TestCompilerFromArgs config @@ -105,4 +109,20 @@ let ``2. Test parallelfs type-checking`` (code: Codebase) = [] let ``3. Test graph-based type-checking`` (code: Codebase) = let config = codebaseToConfig code Method.Graph + OptimizeInputs.optimizerMode <- OptimizerMode.Sequential TestCompilerFromArgs config + +/// Before running this test, you must prepare the codebase by running the script 'FCS.prepare.ps1' +[] +let ``4. Test graph-based type-checking - partially parallel opt`` (code: Codebase) = + let config = codebaseToConfig code Method.Graph + OptimizeInputs.optimizerMode <- OptimizerMode.PartiallyParallel + TestCompilerFromArgs config + +/// Before running this test, you must prepare the codebase by running the script 'FCS.prepare.ps1' +[] +let ``4. Test graph-based type-checking - graph-based opt`` (code: Codebase) = + let config = codebaseToConfig code Method.Graph + OptimizeInputs.optimizerMode <- OptimizerMode.GraphBased + TestCompilerFromArgs config + OptimizeInputs.goer <- Some GraphBasedOpt.goGraph \ No newline at end of file diff --git a/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs b/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs index acac7384266..8cabf269083 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs @@ -25,11 +25,15 @@ let codebases = WorkDir = $@"{__SOURCE_DIRECTORY__}\.fcs_test\tests\FSharp.Compiler.ComponentTests" Path = $@"{__SOURCE_DIRECTORY__}\ComponentTests.args.txt" } - // Hard coded example ;) - // { - // WorkDir = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core" - // Path = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\args.txt" - // } + // Hard coded example ;) + { + WorkDir = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core" + Path = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\args.txt" + } + { + WorkDir = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core.Tests" + Path = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core.Tests\args.txt" + } |] let checker = FSharpChecker.Create(keepAssemblyContents = true) @@ -38,9 +42,14 @@ type DepCollector(projectRoot: string, projectFile: string) = let deps = HashSet() member this.Add(declarationLocation: range) : unit = - let sourceLocation = declarationLocation.FileName + let sourceLocation = Path.GetFullPath declarationLocation.FileName + let ext = Path.GetExtension sourceLocation - if sourceLocation.StartsWith projectRoot && sourceLocation <> projectFile then + if + (ext = ".fs" || ext = ".fsi") + && sourceLocation.StartsWith projectRoot + && sourceLocation <> projectFile + then deps.Add(sourceLocation.Substring(projectRoot.Length + 1)) |> ignore member this.Deps = Seq.toArray deps @@ -76,7 +85,7 @@ let graphFromTypedTree (checker: FSharpChecker) (projectDir: string) (projectOptions: FSharpProjectOptions) - : Async * IReadOnlyDictionary> = + : Async * DepsGraph> = async { let files = Dictionary() @@ -96,7 +105,8 @@ let graphFromTypedTree | FSharpCheckFileAnswer.Aborted -> return failwith "aborted" | FSharpCheckFileAnswer.Succeeded fileResult -> let allSymbols = fileResult.GetAllUsesOfAllSymbolsInFile() |> Seq.toArray - let collector = DepCollector(projectDir, fileName) + let fullPath = Path.GetFullPath fileName + let collector = DepCollector(projectDir, fullPath) for s in allSymbols do collectFromSymbol collector s.Symbol @@ -131,7 +141,6 @@ let ``Create Graph from typed tree`` (code: Codebase) = let previousDir = Environment.CurrentDirectory async { - try Environment.CurrentDirectory <- code.WorkDir @@ -173,6 +182,9 @@ let ``Create Graph from typed tree`` (code: Codebase) = let path = $"{fileName}.typed-tree.deps.json" graphFromTypedTree |> Graph.map (fun n -> n.Name) |> Graph.serialiseToJson path + let fileIndexToName = + files.Values |> Seq.map (fun file -> file.Idx.Idx, file.Name) |> dict + let sourceFiles = files.Values |> Seq.sortBy (fun file -> file.Idx.Idx) @@ -194,25 +206,103 @@ let ``Create Graph from typed tree`` (code: Codebase) = Assert.True(graphFromTypedTree.Count = graphFromHeuristic.Graph.Count, "Both graphs should have the same amount of entries.") - let depNames (files: File array) = - Array.map (fun (f: File) -> Path.GetFileName(f.Name)) files + let depNames (depIdxs: int seq) = + depIdxs + |> Seq.map (fun idx -> fileIndexToName.[idx] |> Path.GetFileName) |> String.concat ", " - for KeyValue (file, deps) in graphFromHeuristic.Graph do - let depsFromTypedTree = graphFromTypedTree.[file] + let collectAllDeps (graph: DepsGraph) = + let getKeyByIdx idx = + graph.Keys |> Seq.find (fun file -> file.Idx.Idx = idx) + + (Map.empty, [ 0 .. (sourceFiles.Length - 1) ]) + ||> List.fold (fun acc idx -> + let deps = graph.[getKeyByIdx idx] + + let allDeps = + set + [| + yield! (Seq.map (fun dep -> dep.Idx.Idx) deps) + yield! (Seq.collect (fun dep -> Map.find dep.Idx.Idx acc) deps) + |] + + Map.add idx allDeps acc) - if Array.isEmpty depsFromTypedTree && not (Array.isEmpty deps) then - printfn $"{file.Name} has %A{(depNames deps)} while the typed tree had none!" + let typedTreeMap = collectAllDeps graphFromTypedTree + let heuristicMap = collectAllDeps graphFromHeuristic.Graph + + /// Compare the found dependencies of a specified heuristic versus the dependencies found in the typed tree + let compareDeps source fileName idx (depsFromHeuristic: Set) = + let depsFromTypedTree = Map.find idx typedTreeMap + + if Set.isEmpty depsFromTypedTree && not (Set.isEmpty depsFromHeuristic) then + printfn $"{source}:{fileName} has %A{(depNames depsFromHeuristic)} while the typed tree had none!" else - let isSuperSet = - depsFromTypedTree |> Seq.forall (fun ttDep -> Seq.contains ttDep deps) + let isSuperSet = Set.isSuperset depsFromHeuristic depsFromTypedTree + let delta = Set.difference depsFromTypedTree depsFromHeuristic Assert.IsTrue( isSuperSet, - $"""{file.Name} did not contain a superset of the typed tree dependencies: -Typed tree dependencies: %A{depNames depsFromTypedTree}. -Heuristic dependencies: %A{depNames deps}.""" + $"""{fileName} did not contain a superset of the typed tree dependencies: +{source} is missing dependencies: %A{depNames delta}.""" ) + + graphFromHeuristic.Graph.Keys + |> Seq.toArray + |> Array.Parallel.iter (fun file -> compareDeps "Current heuristic" file.Name file.Idx.Idx (Map.find file.Idx.Idx heuristicMap)) + + let alternativeGraphFromHeuristic = + let sourceFiles = + files.Values + |> Seq.sortBy (fun file -> file.Idx.Idx) + |> Seq.map (fun file -> + let ast = + match file.AST with + | ASTOrFsix.AST ast -> ast + | ASTOrFsix.Fsix _ -> failwith "unexpected fsix" + + { + Idx = file.Idx.Idx + File = file.Name + AST = ast + }: ParallelTypeCheckingTests.Code.TrieApproach.FileWithAST) + |> Seq.toArray + + ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution.mkGraph sourceFiles + + let alternateMap = + let getDepsByIdx idx = + alternativeGraphFromHeuristic + |> Seq.find (fun (file, _) -> file.Idx = idx) + |> snd + + (Map.empty, [ 0 .. (sourceFiles.Length - 1) ]) + ||> List.fold (fun acc idx -> + let deps = getDepsByIdx idx + + let allDeps = + set [| yield! deps; yield! (Seq.collect (fun dep -> Map.find dep acc) deps) |] + + Map.add idx allDeps acc) + + let rec getAllDepsFromAlt idx : Set = + let currentDeps = + alternativeGraphFromHeuristic + |> Seq.find (fun (file, _) -> file.Idx = idx) + |> snd + + let transitiveDeps = Seq.collect getAllDepsFromAltMemoized currentDeps + + set [| yield! currentDeps; yield! transitiveDeps |] + + and getAllDepsFromAltMemoized = + Internal.Utilities.Library.Tables.memoize getAllDepsFromAlt + + Array.Parallel.iter + (fun (file: ParallelTypeCheckingTests.Code.TrieApproach.FileWithAST, _) -> + compareDeps "Alternative heuristic" file.File file.Idx (Map.find file.Idx alternateMap)) + alternativeGraphFromHeuristic + finally Environment.CurrentDirectory <- previousDir }