From f9c343a0f3608a48f902bb5316c223fbd0b216f5 Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 15 Nov 2022 14:34:44 +0100 Subject: [PATCH 01/21] Initial trie approach. --- .../Code/TrieApproach.fs | 1013 +++++++++++++++++ .../ParallelTypeCheckingTests.fsproj | 1 + 2 files changed, 1014 insertions(+) create mode 100644 tests/ParallelTypeCheckingTests/Code/TrieApproach.fs diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach.fs new file mode 100644 index 0000000000..a3ae7d4e3e --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach.fs @@ -0,0 +1,1013 @@ +module ParallelTypeCheckingTests.Code.TrieApproach + +open System.Collections.Generic +open NUnit.Framework + +// This is pseudo code of how we could restructure the trie code +// My main benefit is that you can easily visually inspect if an identifier will match something in the trie + +type File = string +type Files = Set +type ModuleSegment = string + +/// There is a subtle difference a module and namespace. +/// A namespace does not necessarily expose a set of dependent files. +/// Only when the namespace exposes types that could later be inferred. +/// Children of a namespace don't automatically depend on each other for that reason +type TrieNodeInfo = + | Root + | Module of segment: string * file: File + | Namespace of segment: string * filesThatExposeTypes: Files + + member x.Segment = + match x with + | Root -> failwith "Root has no segment" + | Module (segment = segment) + | Namespace (segment = segment) -> segment + + member x.Files: Files = + match x with + | Root -> failwith "Root has no files" + | Module (file = file) -> Set.singleton file + | Namespace (filesThatExposeTypes = files) -> set files + +type TrieNode = + { + Current: TrieNodeInfo + Children: IDictionary + } + + member x.Files = x.Current.Files + +type FileContentEntry = + /// Any toplevel namespace a file might have. + /// In case a file has `module X.Y.Z`, then `X.Y` is considered to be the toplevel namespace + | TopLevelNamespace of path: ModuleSegment list * content: FileContentEntry list + /// The `open X.Y.Z` syntax. + | OpenStatement of path: ModuleSegment list + /// Any identifier that has more than one piece (LongIdent or SynLongIdent) in it. + /// The last part of the identifier should not be included. + | PrefixedIdentifier of path: ModuleSegment list + /// Being explicit about nested modules allows for easier reasoning what namespaces (paths) are open. + /// We can scope an `OpenStatement` to the everything that is happening inside the nested module. + | NestedModule of name: string * nestedContent: FileContentEntry list + +type FileContent = + { + Name: File + Idx: int + Content: FileContentEntry array + } + +type FileContentQueryState = + { + OpenNamespaces: Set + FoundDependencies: Set + CurrentFile: File + KnownFiles: Files + } + + static member Create (file: File) (knownFiles: Files) = + { + OpenNamespaces = Set.empty + FoundDependencies = Set.empty + CurrentFile = file + KnownFiles = knownFiles + } + + member x.AddDependencies(files: Files) : FileContentQueryState = + let files = Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies + { x with FoundDependencies = files } + + member x.AddOpenNamespace(path: ModuleSegment list) = + { x with + OpenNamespaces = Set.add path x.OpenNamespaces + } + + member x.AddDependenciesAndOpenNamespace(files: Files, path: ModuleSegment list) = + let foundDependencies = + Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies + + { x with + FoundDependencies = foundDependencies + OpenNamespaces = Set.add path x.OpenNamespaces + } + +[] +type QueryTrieNodeResult = + /// No node was found for the path in the trie + | NodeDoesNotExist + /// A node was found but it yielded no file links + | NodeDoesNotExposeData + /// A node was found with one or more file links + | NodeExposesData of Files + +// This code just looks for a path in the trie +// It could be cached and is easy to reason about. +let queryTrie (trie: TrieNode) (path: ModuleSegment list) : QueryTrieNodeResult = + let rec visit (currentNode: TrieNode) (path: ModuleSegment list) = + match path with + | [] -> failwith "path should not be empty" + | [ lastNodeFromPath ] -> + let childResults = + currentNode.Children + |> Seq.tryFind (fun (KeyValue (segment, _childNode)) -> segment = lastNodeFromPath) + + match childResults with + | None -> QueryTrieNodeResult.NodeDoesNotExist + | Some (KeyValue (_, childNode)) -> + if Set.isEmpty childNode.Files then + QueryTrieNodeResult.NodeDoesNotExposeData + else + QueryTrieNodeResult.NodeExposesData(childNode.Files) + | currentPath :: restPath -> + let childResults = + currentNode.Children + |> Seq.tryFind (fun (KeyValue (segment, _childNode)) -> segment = currentPath) + + match childResults with + | None -> QueryTrieNodeResult.NodeDoesNotExist + | Some (KeyValue (_, childNode)) -> visit childNode restPath + + visit trie path + +let noChildren = dict [||] + +// This should be constructed from the AST +let fantomasCoreTrie: TrieNode = + { + Current = TrieNodeInfo.Root + Children = + dict + [| + "System", + { + Current = TrieNodeInfo.Namespace("System", Set.empty) + Children = + dict + [| + "AssemblyVersionInformation", + { + Current = TrieNodeInfo.Module("AssemblyVersionInformation", "AssemblyInfo.fs") + Children = noChildren + } + |] + } + "Fantomas", + { + Current = TrieNodeInfo.Namespace("Fantomas", Set.empty) + Children = + dict + [| + "Core", + { + Current = TrieNodeInfo.Namespace("Core", Set.empty) + Children = + dict + [| + "ISourceTextExtensions", + { + Current = TrieNodeInfo.Module("ISourceTextExtensions", "ISourceTextExtensions.fs") + Children = noChildren + } + "RangeHelpers", + { + Current = TrieNodeInfo.Module("RangeHelpers", "RangeHelpers.fs") + Children = noChildren + } + "RangePatterns", + { + Current = TrieNodeInfo.Module("RangePatterns", "RangeHelpers.fs") + Children = noChildren + } + "AstExtensions", + { + Current = TrieNodeInfo.Module("AstExtensions", "AstExtensions.fs") + Children = noChildren + } + "TriviaTypes", + { + Current = TrieNodeInfo.Module("TriviaTypes", "TriviaTypes.fs") + Children = noChildren + } + "Char", + { + Current = TrieNodeInfo.Module("Char", "Utils.fs") + Children = noChildren + } + "String", + { + Current = TrieNodeInfo.Module("String", "Utils.fs") + Children = noChildren + } + "Cache", + { + Current = TrieNodeInfo.Module("Cache", "Utils.fs") + Children = noChildren + } + "Dict", + { + Current = TrieNodeInfo.Module("Dict", "Utils.fs") + Children = noChildren + } + "List", + { + Current = TrieNodeInfo.Module("List", "Utils.fs") + Children = noChildren + } + "Map", + { + Current = TrieNodeInfo.Module("Map", "Utils.fs") + Children = noChildren + } + "Async", + { + Current = TrieNodeInfo.Module("Async", "Utils.fs") + Children = noChildren + } + "Continuation", + { + Current = TrieNodeInfo.Module("Continuation", "Utils.fs") + Children = noChildren + } + "SourceParser", + { + Current = TrieNodeInfo.Module("SourceParser", "SourceParser.fs") + Children = noChildren + } + |] + } + |] + } + |] + } + +// Some helper DSL functions to construct the FileContentEntry items +// This should again be mapped from the AST + +let topLevelNS (topLevelNamespaceString: string) (content: FileContentEntry list) = + topLevelNamespaceString.Split(".") + |> Array.toList + |> fun name -> FileContentEntry.TopLevelNamespace(name, content) + +let topLevelMod (topLevelModuleString: string) (content: FileContentEntry list) = + let parts = topLevelModuleString.Split(".") + + parts + |> Array.take (parts.Length - 1) + |> Array.toList + |> fun name -> FileContentEntry.TopLevelNamespace(name, content) + +let openSt (openStatement: string) = + openStatement.Split(".") |> Array.toList |> FileContentEntry.OpenStatement + +let nestedModule name content = + FileContentEntry.NestedModule(name, content) + +let prefIdent (lid: string) = + let parts = lid.Split(".") + Array.take (parts.Length - 1) parts |> List.ofArray |> PrefixedIdentifier + +// Some hardcoded files processing, this was done by the naked eye and some regexes. + +let files = + [| + { + Name = "AssemblyInfo.fs" + Idx = 0 + Content = + [| + topLevelNS + "System" + [ + openSt "System.Runtime.CompilerServices" + nestedModule "AssemblyVersionInformation" [] + ] + |] + } + { + Name = "ISourceTextExtensions.fs" + Idx = 1 + Content = + [| + topLevelMod + "Fantomas.Core.ISourceTextExtensions" + [ + openSt "System.Text" + openSt "FSharp.Compiler.Text" + prefIdent "range.StartLine" + prefIdent "this.GetLineString" + prefIdent "range.StartLine" + prefIdent "range.EndLine" + prefIdent "range.EndColumn" + prefIdent "range.StartColumn" + prefIdent "line.Substring" + prefIdent "sb.AppendLine" + prefIdent "lastLine.Substring" + ] + |] + } + { + Name = "RangeHelpers.fs" + Idx = 2 + Content = + [| + topLevelNS + "Fantomas.Core" + [ + openSt "FSharp.Compiler.Text" + nestedModule + "RangeHelpers" + [ + prefIdent "Position.posGeq" + prefIdent "b.Start" + prefIdent "a.Start" + prefIdent "a.End" + prefIdent "b.End" + prefIdent "Range.equals" + prefIdent "r1.FileName" + prefIdent "r2.FileName" + prefIdent "r1.End" + prefIdent "r2.Start" + prefIdent "r1.EndColumn" + prefIdent "r2.StartColumn" + prefIdent "Range.mkRange" + prefIdent "r.FileName" + prefIdent "r.Start" + prefIdent "Position.mkPos" + prefIdent "r.StartLine" + prefIdent "r.StartColumn" + prefIdent "r.EndLine" + prefIdent "r.EndColumn" + prefIdent "r.End" + prefIdent "List.sortBy" + prefIdent "List.reduce" + prefIdent "Range.unionRanges" + ] + nestedModule + "RangePatterns" + [ + prefIdent "RangeHelpers.mkStartEndRange" + prefIdent "range.FileName" + prefIdent "range.Start" + prefIdent "range.StartLine" + prefIdent "range.StartColumn" + ] + ] + |] + } + { + Name = "AstExtensions.fsi" + Idx = 3 + Content = + [| + topLevelMod "Fantomas.Core.AstExtensions" [ openSt "FSharp.Compiler.Text"; openSt "FSharp.Compiler.Syntax" ] + |] + } + { + Name = "AstExtensions.fs" + Idx = 4 + Content = + [| + topLevelMod + "Fantomas.Core.AstExtensions" + [ + openSt "FSharp.Compiler.SyntaxTrivia" + openSt "FSharp.Compiler.Text" + openSt "FSharp.Compiler.Text.Range" + openSt "FSharp.Compiler.Syntax" + prefIdent "range.Zero" + prefIdent "h.idRange" + prefIdent "List.last" + prefIdent "ident.idRange" + prefIdent "IdentTrivia.OriginalNotationWithParen" + prefIdent "IdentTrivia.HasParenthesis" + prefIdent "IdentTrivia.OriginalNotation" + prefIdent "Range.Zero" + prefIdent "single.FullRange" + prefIdent "List.fold" + prefIdent "head.FullRange" + prefIdent "fieldName.FullRange" + prefIdent "expr.Range" + prefIdent "SynModuleOrNamespaceKind.AnonModule" + prefIdent "List.tryHead" + prefIdent "List.tryLast" + prefIdent "d.Range" + prefIdent "s.Range" + prefIdent "e.Range" + prefIdent "this.Range" + prefIdent "CommentTrivia.LineComment" + prefIdent "CommentTrivia.BlockComment" + prefIdent "ConditionalDirectiveTrivia.If" + prefIdent "ConditionalDirectiveTrivia.Else" + prefIdent "ConditionalDirectiveTrivia.EndIf" + prefIdent "List.map" + prefIdent "c.Range" + prefIdent "acc.StartLine" + prefIdent "triviaRange.StartLine" + prefIdent "acc.EndLine" + prefIdent "triviaRange.EndLine" + prefIdent "ParsedInput.ImplFile" + prefIdent "r.Start" + prefIdent "m.FullRange.Start" + prefIdent "Range.Zero.Start" + prefIdent "Range.Zero.End" + prefIdent "r.End" + prefIdent "lastModule.FullRange.End" + prefIdent "this.Range.FileName" + prefIdent "trivia.CodeComments" + prefIdent "trivia.ConditionalDirectives" + prefIdent "ParsedInput.SigFile" + prefIdent "SynInterpolatedStringPart.String" + prefIdent "SynInterpolatedStringPart.FillExpr" + prefIdent "i.idRange" + prefIdent "std.FullRange" + prefIdent "a.Range" + prefIdent "RangeHelpers.mergeRanges" + prefIdent "synTypar.Range" + prefIdent "sf.FullRange" + prefIdent "head.Range" + prefIdent "b.FullRange" + prefIdent "xmlDoc.IsEmpty" + prefIdent "xmlDoc.Range" + prefIdent "attributes.IsEmpty" + prefIdent "attributes.Head.Range" + prefIdent "trivia.LeadingKeyword" + prefIdent "SynLeadingKeyword.Member" + prefIdent "SynPat.LongIdent" + prefIdent "pat.Range" + prefIdent "trivia.LeadingKeyword.Range" + ] + |] + } + { + Name = "TriviaTypes.fs" + Idx = 5 + Content = + [| + topLevelMod "Fantomas.Core.TriviaTypes" [ openSt "FSharp.Compiler.Text"; openSt "FSharp.Compiler.Syntax" ] + |] + } + { + Name = "Utils.fs" + Idx = 6 + Content = + [| + topLevelNS + "Fantomas.Core" + [ + openSt "System" + openSt "System.Text.RegularExpressions" + nestedModule "Char" [ prefIdent "c.ToString" ] + nestedModule + "String" + [ + prefIdent "str.Replace" + prefIdent "str.StartsWith" + prefIdent "StringComparison.Ordinal" + prefIdent "String.Empty" + prefIdent "source.Split" + prefIdent "StringSplitOptions.None" + prefIdent "Array.mapi" + prefIdent "Regex.IsMatch" + prefIdent "Array.choose" + prefIdent "Array.toList" + prefIdent "List.tryHead" + prefIdent "List.map" + prefIdent "String.concat" + prefIdent "List.zip" + prefIdent "String.length" + prefIdent "String.IsNullOrEmpty" + prefIdent "String.IsNullOrWhiteSpace" + prefIdent "String.exists" + ] + nestedModule + "Cache" + [ + prefIdent "System.Collections.Generic.HashSet" + prefIdent "HashIdentity.Reference" + prefIdent "cache.Contains" + prefIdent "cache.Add" + prefIdent "System.Collections.Concurrent.ConcurrentDictionary" + prefIdent "HashIdentity.Structural" + prefIdent "cache.GetOrAdd" + prefIdent "this.Equals" + prefIdent "Object.ReferenceEquals" + prefIdent "this.GetHashCode" + ] + nestedModule + "Dict" + [ + prefIdent "System.Collections.Generic.IDictionary" + prefIdent "d.TryGetValue" + ] + nestedModule + "List" + [ + prefIdent "List.takeWhile" + prefIdent "List.choose" + prefIdent "List.isEmpty" + prefIdent "List.rev" + ] + nestedModule "Map" [ prefIdent "Map.tryFind" ] + nestedModule "Async" [ prefIdent "async.Bind"; prefIdent "async.Return" ] + nestedModule "Continuation" [] + ] + |] + } + { + Name = "SourceParser.fs" + Idx = 7 + Content = + [| + topLevelMod + "Fantomas.Core.SourceParser" + [ + openSt "System" + openSt "FSharp.Compiler.Syntax" + openSt "FSharp.Compiler.Syntax.PrettyNaming" + openSt "FSharp.Compiler.SyntaxTrivia" + openSt "FSharp.Compiler.Text" + openSt "FSharp.Compiler.Xml" + openSt "Fantomas.Core" + openSt "Fantomas.Core.AstExtensions" + openSt "Fantomas.Core.TriviaTypes" + openSt "Fantomas.Core.RangePatterns" + prefIdent "SynTypar.SynTypar" + prefIdent "TyparStaticReq.None" + prefIdent "TyparStaticReq.HeadType" + prefIdent "SynRationalConst.Integer" + prefIdent "SynRationalConst.Rational" + prefIdent "SynRationalConst.Negate" + prefIdent "SynConst.Unit" + prefIdent "ParsedInput.ImplFile" + prefIdent "ParsedInput.SigFile" + prefIdent "ParsedImplFileInput.ParsedImplFileInput" + prefIdent "ParsedSigFileInput.ParsedSigFileInput" + prefIdent "SynModuleOrNamespace.SynModuleOrNamespace" + prefIdent "trivia.LeadingKeyword" + prefIdent "m.FullRange" + prefIdent "SynModuleOrNamespaceSig.SynModuleOrNamespaceSig" + prefIdent "a.TypeName" + prefIdent "a.ArgExpr" + prefIdent "a.Target" + prefIdent "px.ToXmlDoc" + prefIdent "xmlDoc.UnprocessedLines" + prefIdent "xmlDoc.Range" + prefIdent "SynModuleDecl.Open" + prefIdent "SynOpenDeclTarget.ModuleOrNamespace" + prefIdent "SynOpenDeclTarget.Type" + prefIdent "SynType.LongIdent" + prefIdent "SynModuleDecl.ModuleAbbrev" + prefIdent "SynModuleDecl.HashDirective" + prefIdent "SynModuleDecl.NamespaceFragment" + prefIdent "SynModuleDecl.Attributes" + prefIdent "SynModuleDecl.Let" + prefIdent "SynModuleDecl.Expr" + prefIdent "SynModuleDecl.Types" + prefIdent "SynModuleDecl.NestedModule" + prefIdent "trivia.ModuleKeyword" + prefIdent "trivia.EqualsRange" + prefIdent "SynModuleDecl.Exception" + prefIdent "SynModuleSigDecl.Open" + prefIdent "SynModuleSigDecl.ModuleAbbrev" + prefIdent "SynModuleSigDecl.HashDirective" + prefIdent "SynModuleSigDecl.NamespaceFragment" + prefIdent "SynModuleSigDecl.Val" + prefIdent "SynModuleSigDecl.Types" + prefIdent "SynModuleSigDecl.NestedModule" + prefIdent "SynModuleSigDecl.Exception" + prefIdent "SynExceptionDefnRepr.SynExceptionDefnRepr" + prefIdent "SynExceptionDefn.SynExceptionDefn" + prefIdent "SynExceptionSig.SynExceptionSig" + prefIdent "px.IsEmpty" + prefIdent "trivia.BarRange" + prefIdent "Range.unionRanges" + prefIdent "SynUnionCaseKind.Fields" + prefIdent "SynUnionCaseKind.FullType" + prefIdent "Option.map" + prefIdent "i.idRange" + prefIdent "t.Range" + prefIdent "SynMemberDefn.NestedType" + prefIdent "SynMemberDefn.Open" + prefIdent "SynMemberDefn.ImplicitInherit" + prefIdent "SynMemberDefn.Inherit" + prefIdent "SynMemberDefn.ValField" + prefIdent "SynMemberDefn.ImplicitCtor" + prefIdent "SynMemberDefn.Member" + prefIdent "SynMemberDefn.LetBindings" + prefIdent "SynType.Fun" + prefIdent "SynMemberKind.PropertyGet" + prefIdent "SynMemberKind.PropertySet" + prefIdent "SynMemberKind.PropertyGetSet" + prefIdent "SynMemberDefn.AbstractSlot" + prefIdent "trivia.WithKeyword" + prefIdent "mf.MemberKind" + prefIdent "SynMemberDefn.Interface" + prefIdent "SynMemberDefn.AutoProperty" + prefIdent "SynMemberDefn.GetSetMember" + prefIdent "SynPat.LongIdent" + prefIdent "Position.posLt" + prefIdent "getKeyword.Start" + prefIdent "setKeyword.Start" + prefIdent "SynMemberKind.ClassConstructor" + prefIdent "SynMemberKind.Constructor" + prefIdent "SynMemberKind.Member" + prefIdent "mf.IsInstance" + prefIdent "mf.IsOverrideOrExplicitImpl" + prefIdent "SynExpr.Typed" + prefIdent "RangeHelpers.rangeEq" + prefIdent "t1.Range" + prefIdent "t2.Range" + prefIdent "Option.bind" + prefIdent "trivia.ColonRange" + prefIdent "b.FullRange" + prefIdent "SynBindingKind.Do" + prefIdent "SynLeadingKeyword.Extern" + prefIdent "SynExpr.TraitCall" + prefIdent "SynExpr.Quote" + prefIdent "SynExpr.Paren" + prefIdent "SynExpr.Lazy" + prefIdent "SynExpr.InferredDowncast" + prefIdent "SynExpr.InferredUpcast" + prefIdent "SynExpr.Assert" + prefIdent "SynExpr.AddressOf" + prefIdent "SynExpr.YieldOrReturn" + prefIdent "SynExpr.YieldOrReturnFrom" + prefIdent "SynExpr.Do" + prefIdent "SynExpr.DoBang" + prefIdent "SynExpr.Fixed" + prefIdent "SynExpr.TypeTest" + prefIdent "SynExpr.Downcast" + prefIdent "SynExpr.Upcast" + prefIdent "SynExpr.While" + prefIdent "SynExpr.For" + prefIdent "SynExpr.Null" + prefIdent "SynExpr.Const" + prefIdent "SynExpr.TypeApp" + prefIdent "SynExpr.Match" + prefIdent "trivia.MatchKeyword" + prefIdent "SynExpr.MatchBang" + prefIdent "trivia.MatchBangKeyword" + prefIdent "SynExpr.Sequential" + prefIdent "SynExpr.Ident" + prefIdent "SynExpr.LongIdent" + prefIdent "SynExpr.ComputationExpr" + prefIdent "SynExpr.App" + prefIdent "ExprAtomicFlag.NonAtomic" + prefIdent "compExpr.Range" + prefIdent "SynExpr.ArrayOrListComputed" + prefIdent "RangeHelpers.mkStartEndRange" + prefIdent "SynExpr.ArrayOrList" + prefIdent "SynExpr.Tuple" + prefIdent "SynExpr.InterpolatedString" + prefIdent "SynExpr.IndexRange" + prefIdent "SynExpr.IndexFromEnd" + prefIdent "SynExpr.Typar" + prefIdent "SynConst.Double" + prefIdent "SynConst.Decimal" + prefIdent "SynConst.Single" + prefIdent "SynConst.Int16" + prefIdent "SynConst.Int32" + prefIdent "SynConst.Int64" + prefIdent "List.moreThanOne" + prefIdent "SynExpr.Dynamic" + prefIdent "IdentTrivia.OriginalNotationWithParen" + prefIdent "originalNotation.Length" + prefIdent "originalNotation.StartsWith" + prefIdent "List.rev" + prefIdent "SynExpr.DotGet" + prefIdent "SynExpr.Lambda" + prefIdent "SynExpr.MatchLambda" + prefIdent "SynExpr.New" + prefIdent "IdentTrivia.OriginalNotation" + prefIdent "ident.idText" + prefIdent "newLineInfixOps.Contains" + prefIdent "List.length" + prefIdent "SynExpr.JoinIn" + prefIdent "SynExpr.LetOrUse" + prefIdent "xs.Length" + prefIdent "List.mapi" + prefIdent "trivia.InKeyword" + prefIdent "List.map" + prefIdent "SynExpr.LetOrUseBang" + prefIdent "List.collect" + prefIdent "Continuation.sequence" + prefIdent "SynExpr.ForEach" + prefIdent "SynExpr.DotIndexedSet" + prefIdent "SynExpr.NamedIndexedPropertySet" + prefIdent "SynExpr.DotNamedIndexedPropertySet" + prefIdent "SynExpr.DotIndexedGet" + prefIdent "SynExpr.DotSet" + prefIdent "SynExpr.IfThenElse" + prefIdent "trivia.IfKeyword" + prefIdent "trivia.IsElif" + prefIdent "trivia.ThenKeyword" + prefIdent "trivia.ElseKeyword" + prefIdent "unitRange.StartColumn" + prefIdent "unitRange.EndColumn" + prefIdent "SynExpr.Record" + prefIdent "SynExpr.AnonRecd" + prefIdent "SynExpr.ObjExpr" + prefIdent "SynExpr.LongIdentSet" + prefIdent "SynExpr.TryWith" + prefIdent "trivia.TryKeyword" + prefIdent "SynExpr.TryFinally" + prefIdent "trivia.FinallyKeyword" + prefIdent "SynExpr.ArbitraryAfterError" + prefIdent "SynExpr.FromParseError" + prefIdent "SynExpr.DiscardAfterMissingQualificationAfterDot" + prefIdent "SynExpr.LibraryOnlyILAssembly" + prefIdent "SynExpr.LibraryOnlyStaticOptimization" + prefIdent "FSharp.Core" + prefIdent "SynExpr.LibraryOnlyUnionCaseFieldGet" + prefIdent "SynExpr.LibraryOnlyUnionCaseFieldSet" + prefIdent "SynPat.OptionalVal" + prefIdent "SynPat.Attrib" + prefIdent "SynPat.Or" + prefIdent "p.Range" + prefIdent "SynPat.Ands" + prefIdent "SynPat.Null" + prefIdent "SynPat.Wild" + prefIdent "SynPat.Tuple" + prefIdent "SynPat.ArrayOrList" + prefIdent "SynPat.Typed" + prefIdent "SynPat.Named" + prefIdent "SynPat.As" + prefIdent "SynArgPats.NamePatPairs" + prefIdent "SynArgPats.Pats" + prefIdent "SynPat.ListCons" + prefIdent "trivia.ColonColonRange" + prefIdent "synLongIdent.IdentsWithTrivia" + prefIdent "synIdent.FullRange" + prefIdent "synLongIdent.FullRange" + prefIdent "SynPat.Paren" + prefIdent "SynPat.Record" + prefIdent "SynPat.Const" + prefIdent "SynPat.IsInst" + prefIdent "SynPat.QuoteExpr" + prefIdent "newIdent.idText" + prefIdent "pat.Range" + prefIdent "SynSimplePats.SimplePats" + prefIdent "SynSimplePats.Typed" + prefIdent "SynSimplePat.Attrib" + prefIdent "SynSimplePat.Id" + prefIdent "SynSimplePat.Typed" + prefIdent "trivia.ArrowRange" + prefIdent "SynMatchClause.SynMatchClause" + prefIdent "matchRange.Start" + prefIdent "clause.Range.Start" + prefIdent "me.Range" + prefIdent "SynTypeDefnSimpleRepr.Enum" + prefIdent "SynTypeDefnSimpleRepr.Union" + prefIdent "SynTypeDefnSimpleRepr.Record" + prefIdent "SynTypeDefnSimpleRepr.None" + prefIdent "SynTypeDefnSimpleRepr.TypeAbbrev" + prefIdent "SynTypeDefnSimpleRepr.General" + prefIdent "SynTypeDefnSimpleRepr.LibraryOnlyILAssembly" + prefIdent "SynTypeDefnSimpleRepr.Exception" + prefIdent "SynTypeDefnRepr.Simple" + prefIdent "SynTypeDefnRepr.ObjectModel" + prefIdent "SynTypeDefnRepr.Exception" + prefIdent "List.tryFind" + prefIdent "List.filter" + prefIdent "SynTypeDefnSigRepr.Simple" + prefIdent "SynTypeDefnSigRepr.ObjectModel" + prefIdent "SynTypeDefnSigRepr.Exception" + prefIdent "SynTypeDefnKind.Unspecified" + prefIdent "SynTypeDefnKind.Class" + prefIdent "SynTypeDefnKind.Interface" + prefIdent "SynTypeDefnKind.Struct" + prefIdent "SynTypeDefnKind.Record" + prefIdent "SynTypeDefnKind.Union" + prefIdent "SynTypeDefnKind.Abbrev" + prefIdent "SynTypeDefnKind.Opaque" + prefIdent "SynTypeDefnKind.Augmentation" + prefIdent "SynTypeDefnKind.IL" + prefIdent "SynTypeDefnKind.Delegate" + prefIdent "std.FullRange" + prefIdent "SynTyparDecls.PostfixList" + prefIdent "SynType.HashConstraint" + prefIdent "SynType.MeasurePower" + prefIdent "SynType.MeasureDivide" + prefIdent "SynType.StaticConstant" + prefIdent "SynType.StaticConstantExpr" + prefIdent "SynType.StaticConstantNamed" + prefIdent "SynType.Array" + prefIdent "SynType.Anon" + prefIdent "SynType.Var" + prefIdent "SynType.App" + prefIdent "SynType.LongIdentApp" + prefIdent "SynType.Tuple" + prefIdent "SynType.WithGlobalConstraints" + prefIdent "SynType.AnonRecd" + prefIdent "SynType.Paren" + prefIdent "SynType.SignatureParameter" + prefIdent "SynType.Or" + prefIdent "trivia.OrKeyword" + prefIdent "lid.idText" + prefIdent "x.ToString" + prefIdent "SynTypeConstraint.WhereTyparIsValueType" + prefIdent "SynTypeConstraint.WhereTyparIsReferenceType" + prefIdent "SynTypeConstraint.WhereTyparIsUnmanaged" + prefIdent "SynTypeConstraint.WhereTyparSupportsNull" + prefIdent "SynTypeConstraint.WhereTyparIsComparable" + prefIdent "SynTypeConstraint.WhereTyparIsEquatable" + prefIdent "SynTypeConstraint.WhereTyparDefaultsToType" + prefIdent "SynTypeConstraint.WhereTyparSubtypeOfType" + prefIdent "SynTypeConstraint.WhereTyparSupportsMember" + prefIdent "SynTypeConstraint.WhereTyparIsEnum" + prefIdent "SynTypeConstraint.WhereTyparIsDelegate" + prefIdent "SynTypeConstraint.WhereSelfConstrained" + prefIdent "SynMemberSig.Member" + prefIdent "SynMemberSig.Interface" + prefIdent "SynMemberSig.Inherit" + prefIdent "SynMemberSig.ValField" + prefIdent "SynMemberSig.NestedType" + prefIdent "ident.idRange" + prefIdent "e.Range" + prefIdent "List.tryLast" + prefIdent "IdentTrivia.HasParenthesis" + prefIdent "lp.idText" + prefIdent "Seq.tryHead" + prefIdent "Char.IsUpper" + prefIdent "Option.defaultValue" + prefIdent "ExprAtomicFlag.Atomic" + prefIdent "RangeHelpers.isAdjacentTo" + prefIdent "identifierExpr.Range" + prefIdent "argExpr.Range" + prefIdent "Seq.toList" + prefIdent "Seq.singleton" + prefIdent "List.exists" + ] + |] + } + |] + +// Now how to detect the deps between files? +// Process the content of each file using some state + +// Helper function to process a open statement +// The statement could link to files and/or should be tracked as an open namespace +let processOpenPath (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = + let queryResult = queryTrie fantomasCoreTrie path + + match queryResult with + | QueryTrieNodeResult.NodeDoesNotExist -> state + | QueryTrieNodeResult.NodeDoesNotExposeData -> state.AddOpenNamespace path + | QueryTrieNodeResult.NodeExposesData files -> state.AddDependenciesAndOpenNamespace(files, path) + +// Helper function to process an identifier +let processIdentifier (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = + let queryResult = queryTrie fantomasCoreTrie path + + match queryResult with + | QueryTrieNodeResult.NodeDoesNotExist -> state + | QueryTrieNodeResult.NodeDoesNotExposeData -> failwith "This identifier cannot be part of a node that doesn't expose data!" + | QueryTrieNodeResult.NodeExposesData files -> state.AddDependencies files + +// Typically used to folder FileContentEntry items over a FileContentQueryState +let rec processStateEntry (state: FileContentQueryState) (entry: FileContentEntry) : FileContentQueryState = + match entry with + | FileContentEntry.TopLevelNamespace (topLevelPath, content) -> + let state = + match topLevelPath with + | [] -> state + | _ -> state.AddOpenNamespace topLevelPath + + List.fold processStateEntry state content + + | FileContentEntry.OpenStatement path -> + // An open statement can directly reference file or be a partial open statement + // Both cases need to be processed. + let stateAfterFullOpenPath = processOpenPath path state + + // Any existing open statement could be extended with the current path (if that node where to exists in the trie) + // The extended path could add a new link (in case of a module or namespace with types) + // It might also not add anything at all (in case it the extended path is still a partial one) + (stateAfterFullOpenPath, state.OpenNamespaces) + ||> Seq.fold (fun acc openNS -> processOpenPath [ yield! openNS; yield! path ] acc) + + | FileContentEntry.PrefixedIdentifier path -> + // process the name was if it were a FQN + let stateAfterFullIdentifier = processIdentifier path state + + // Process the name in combination with the existing open namespaces + (stateAfterFullIdentifier, state.OpenNamespaces) + ||> Seq.fold (fun acc openNS -> processIdentifier [ yield! openNS; yield! path ] acc) + + | FileContentEntry.NestedModule (nestedContent = nestedContent) -> + // We don't want our current state to be affect by any open statements in the nested module + let nestedState = List.fold processStateEntry state nestedContent + // Afterward we are only interested in the found dependencies in the nested module + let foundDependencies = + Set.union state.FoundDependencies nestedState.FoundDependencies + + { state with + FoundDependencies = foundDependencies + } + +let getFileNameBefore idx = + files.[0 .. (idx - 1)] |> Array.map (fun f -> f.Name) |> Set.ofArray + +let mkGraph files = + files + |> Array.map (fun (file: FileContent) -> + let knownFiles = getFileNameBefore file.Idx + + let result = + Array.fold processStateEntry (FileContentQueryState.Create file.Name knownFiles) file.Content + + file.Name, Set.toArray result.FoundDependencies) + +[] +let ``Full project simulation`` () = + let graph = mkGraph files + + for fileName, deps in graph do + let depString = String.concat ", " deps + printfn $"%s{fileName}: [{depString}]" + + () + +[] +let ``SourceParser.fs simulation`` () = + let fileName = "SourceParser.fs" + let file = Array.find (fun (fc: FileContent) -> fc.Name = fileName) files + let knownFiles = getFileNameBefore file.Idx + + let result = + Array.fold processStateEntry (FileContentQueryState.Create file.Name knownFiles) file.Content + + let deps = Seq.sort result.FoundDependencies |> Seq.toList + + match deps with + | [ "AstExtensions.fs"; "RangeHelpers.fs"; "TriviaTypes.fs"; "Utils.fs" ] -> Assert.Pass() + | deps -> Assert.Fail $"Unexpected deps for {fileName}, got %A{deps}" + +[] +let ``AstExtensions.fs simulation`` () = + let fileName = "AstExtensions.fs" + let file = Array.find (fun (fc: FileContent) -> fc.Name = fileName) files + let knownFiles = getFileNameBefore file.Idx + + let result = + Array.fold processStateEntry (FileContentQueryState.Create file.Name knownFiles) file.Content + + let deps = Seq.sort result.FoundDependencies |> Seq.toList + + match deps with + | [ "RangeHelpers.fs" ] -> Assert.Pass() + | deps -> Assert.Fail $"Unexpected deps for {fileName}, got %A{deps}" + +[] +let ``Query non existing node in trie`` () = + let result = + queryTrie fantomasCoreTrie [ "System"; "System"; "Runtime"; "CompilerServices" ] + + match result with + | QueryTrieNodeResult.NodeDoesNotExist -> Assert.Pass() + | result -> Assert.Fail $"Unexpected result: %A{result}" + +[] +let ``Query node that does not expose data in trie`` () = + let result = queryTrie fantomasCoreTrie [ "Fantomas"; "Core" ] + + match result with + | QueryTrieNodeResult.NodeDoesNotExposeData -> Assert.Pass() + | result -> Assert.Fail $"Unexpected result: %A{result}" + +[] +let ``Query module node that exposes one file`` () = + let result = + queryTrie fantomasCoreTrie [ "Fantomas"; "Core"; "ISourceTextExtensions" ] + + match result with + | QueryTrieNodeResult.NodeExposesData file -> + let file = Seq.exactlyOne file + Assert.AreEqual("ISourceTextExtensions.fs", file) + | result -> Assert.Fail $"Unexpected result: %A{result}" + +[] +let ``ProcessOpenStatement full path match`` () = + let sourceParser = + Array.find (fun (f: FileContent) -> f.Name = "SourceParser.fs") files + + let state = + FileContentQueryState.Create sourceParser.Name (getFileNameBefore sourceParser.Idx) + + let result = processOpenPath [ "Fantomas"; "Core"; "AstExtensions" ] state + let dep = Seq.exactlyOne result.FoundDependencies + Assert.AreEqual("AstExtensions.fs", dep) + +#if INTERACTIVE +open System.Text.RegularExpressions + +let fileContent = + System.IO.File.ReadAllText(@"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceParser.fs") + +Regex.Matches(fileContent, "(\\w)+(\\.(\\w)+)+") +|> Seq.cast +|> Seq.distinctBy (fun m -> m.Value) +|> Seq.iter (fun m -> printfn "prefIdent \"%s\"" m.Value) +#endif diff --git a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj index 745bef6b0c..2990980afa 100644 --- a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj +++ b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj @@ -37,6 +37,7 @@ + From 23a59469d1f24eb14e3fbe7fe184349c2376c597 Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 17 Nov 2022 11:57:39 +0100 Subject: [PATCH 02/21] Dependency resolution with new trie approach. --- .../Code/TrieApproach/DependencyResolution.fs | 188 +++++ .../Code/TrieApproach/FileContentMapping.fs | 688 ++++++++++++++++++ .../SampleData.fs} | 288 +------- .../Code/TrieApproach/TrieMapping.fs | 271 +++++++ .../Code/TrieApproach/Types.fs | 107 +++ .../ParallelTypeCheckingTests.fsproj | 6 +- 6 files changed, 1298 insertions(+), 250 deletions(-) create mode 100644 tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs create mode 100644 tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs rename tests/ParallelTypeCheckingTests/Code/{TrieApproach.fs => TrieApproach/SampleData.fs} (80%) create mode 100644 tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs create mode 100644 tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs new file mode 100644 index 0000000000..8acb6c2256 --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs @@ -0,0 +1,188 @@ +module ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution + +open FSharp.Compiler.Syntax + +// This is pseudo code of how we could restructure the trie code +// My main benefit is that you can easily visually inspect if an identifier will match something in the trie + +// This code just looks for a path in the trie +// It could be cached and is easy to reason about. +let queryTrie (trie: TrieNode) (path: ModuleSegment list) : QueryTrieNodeResult = + let rec visit (currentNode: TrieNode) (path: ModuleSegment list) = + match path with + | [] -> failwith "path should not be empty" + | [ lastNodeFromPath ] -> + let childResults = + currentNode.Children + |> Seq.tryFind (fun (KeyValue (segment, _childNode)) -> segment = lastNodeFromPath) + + match childResults with + | None -> QueryTrieNodeResult.NodeDoesNotExist + | Some (KeyValue (_, childNode)) -> + if Set.isEmpty childNode.Files then + QueryTrieNodeResult.NodeDoesNotExposeData + else + QueryTrieNodeResult.NodeExposesData(childNode.Files) + | currentPath :: restPath -> + let childResults = + currentNode.Children + |> Seq.tryFind (fun (KeyValue (segment, _childNode)) -> segment = currentPath) + + match childResults with + | None -> QueryTrieNodeResult.NodeDoesNotExist + | Some (KeyValue (_, childNode)) -> visit childNode restPath + + visit trie path + +// Now how to detect the deps between files? +// Process the content of each file using some state + +// Helper function to process a open statement +// The statement could link to files and/or should be tracked as an open namespace +let processOpenPath (trie: TrieNode) (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = + let queryResult = queryTrie trie path + + match queryResult with + | QueryTrieNodeResult.NodeDoesNotExist -> state + | QueryTrieNodeResult.NodeDoesNotExposeData -> state.AddOpenNamespace path + | QueryTrieNodeResult.NodeExposesData files -> state.AddDependenciesAndOpenNamespace(files, path) + +// Helper function to process an identifier +let processIdentifier (trie: TrieNode) (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = + let queryResult = queryTrie trie path + + match queryResult with + | QueryTrieNodeResult.NodeDoesNotExist -> state + | QueryTrieNodeResult.NodeDoesNotExposeData -> + // This can occur when you are have a file that uses a known namespace (for example namespace System). + // When any other code uses that System namespace it won't find anything in the user code. + state + | QueryTrieNodeResult.NodeExposesData files -> state.AddDependencies files + +// Typically used to folder FileContentEntry items over a FileContentQueryState +let rec processStateEntry (trie: TrieNode) (state: FileContentQueryState) (entry: FileContentEntry) : FileContentQueryState = + match entry with + | FileContentEntry.TopLevelNamespace (topLevelPath, content) -> + let state = + match topLevelPath with + | [] -> state + | _ -> processOpenPath trie topLevelPath state + + List.fold (processStateEntry trie) state content + + | FileContentEntry.OpenStatement path -> + // An open statement can directly reference file or be a partial open statement + // Both cases need to be processed. + let stateAfterFullOpenPath = processOpenPath trie path state + + // Any existing open statement could be extended with the current path (if that node where to exists in the trie) + // The extended path could add a new link (in case of a module or namespace with types) + // It might also not add anything at all (in case it the extended path is still a partial one) + (stateAfterFullOpenPath, state.OpenNamespaces) + ||> Seq.fold (fun acc openNS -> processOpenPath trie [ yield! openNS; yield! path ] acc) + + | FileContentEntry.PrefixedIdentifier path -> + // process the name was if it were a FQN + let stateAfterFullIdentifier = processIdentifier trie path state + + // Process the name in combination with the existing open namespaces + (stateAfterFullIdentifier, state.OpenNamespaces) + ||> Seq.fold (fun acc openNS -> processIdentifier trie [ yield! openNS; yield! path ] acc) + + | FileContentEntry.NestedModule (nestedContent = nestedContent) -> + // We don't want our current state to be affect by any open statements in the nested module + let nestedState = List.fold (processStateEntry trie) state nestedContent + // Afterward we are only interested in the found dependencies in the nested module + let foundDependencies = + Set.union state.FoundDependencies nestedState.FoundDependencies + + { state with + FoundDependencies = foundDependencies + } + +let getFileNameBefore (files: FileWithAST array) idx = + files.[0 .. (idx - 1)] |> Array.map (fun f -> f.File) |> Set.ofArray + +let mkGraph (files: FileWithAST array) = + let trie = + let input = + files + |> Array.filter (fun f -> + match f.AST with + | ParsedInput.SigFile _ -> true + | ParsedInput.ImplFile _ -> Array.forall (fun (sigFile: FileWithAST) -> sigFile.File <> $"{f.File}i") files) + + TrieMapping.mkTrie input + + let fileContents = Array.Parallel.map FileContentMapping.mkFileContent files + + files + |> Array.map (fun (file: FileWithAST) -> + let fileContent = fileContents.[file.Idx] + let knownFiles = getFileNameBefore files file.Idx + + let result = + Seq.fold (processStateEntry trie) (FileContentQueryState.Create file.File knownFiles) fileContent + + file.File, Set.toArray result.FoundDependencies) + +// ============================================================================================================= +// ============================================================================================================= + +open NUnit.Framework +open FSharp.Compiler.Service.Tests.Common + +[] +let ``Fantomas.Core for realzies`` () = + let files = + [| + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\Fantomas.Core.AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\ISourceTextExtensions.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\RangeHelpers.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\TriviaTypes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Utils.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceParser.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Version.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Queue.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\FormatConfig.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceTransformer.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Context.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Validation.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fs" + |] + + let filesWithAST = + files + |> Array.mapi (fun idx file -> + { + Idx = idx + AST = parseSourceCode (file, System.IO.File.ReadAllText(file)) + File = file + }) + + let graph = mkGraph filesWithAST + + for fileName, deps in graph do + let depString = String.concat "\n " deps + + if deps.Length = 0 then + printfn $"%s{fileName}: []" + else + printfn $"%s{fileName}:\n {depString}" diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs new file mode 100644 index 0000000000..d2b33de572 --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs @@ -0,0 +1,688 @@ +module rec ParallelTypeCheckingTests.Code.TrieApproach.FileContentMapping + +open FSharp.Compiler.Syntax +open FSharp.Compiler.SyntaxTreeOps + +[] +module Continuation = + let rec sequence<'a, 'ret> (recursions: (('a -> 'ret) -> 'ret) list) (finalContinuation: 'a list -> 'ret) : 'ret = + match recursions with + | [] -> [] |> finalContinuation + | recurse :: recurses -> recurse (fun ret -> sequence recurses (fun rets -> ret :: rets |> finalContinuation)) + +type Continuations = ((FileContentEntry list -> FileContentEntry list) -> FileContentEntry list) list + +let cfo f a = lc f (Option.toList a) +let lc = List.collect + +let identToPath (ident: Ident) = ident.idText + +let longIdentToPath (skipLast: bool) (longId: LongIdent) : ModuleSegment list = + if skipLast then + List.take (longId.Length - 1) longId + else + longId + |> List.map identToPath + +let synLongIdentToPath (skipLast: bool) (synLongIdent: SynLongIdent) = + longIdentToPath skipLast synLongIdent.LongIdent + +let visitSynLongIdent (lid: SynLongIdent) : FileContentEntry list = visitLongIdent lid.LongIdent + +let visitLongIdent (lid: LongIdent) = + match lid with + | [] + | [ _ ] -> [] + | lid -> [ FileContentEntry.PrefixedIdentifier(longIdentToPath true lid) ] + +let visitSynAttribute (a: SynAttribute) : FileContentEntry list = + [ yield! visitSynLongIdent a.TypeName; yield! visitSynExpr a.ArgExpr ] + +let visitSynAttributeList (attributes: SynAttributeList) : FileContentEntry list = + lc visitSynAttribute attributes.Attributes + +let visitSynAttributes (attributes: SynAttributes) : FileContentEntry list = lc visitSynAttributeList attributes + +let visitSynModuleDecl (decl: SynModuleDecl) : FileContentEntry list = + match decl with + | SynModuleDecl.Open(target = SynOpenDeclTarget.ModuleOrNamespace (longId, _)) -> + [ FileContentEntry.OpenStatement(synLongIdentToPath false longId) ] + | SynModuleDecl.Open(target = SynOpenDeclTarget.Type (typeName, _)) -> visitSynType typeName + | SynModuleDecl.Attributes (attributes, _) -> lc visitSynAttributeList attributes + | SynModuleDecl.Expr (expr, _) -> visitSynExpr expr + | SynModuleDecl.NestedModule (moduleInfo = SynComponentInfo (longId = [ ident ]; attributes = attributes); decls = decls) -> + [ + yield! visitSynAttributes attributes + yield FileContentEntry.NestedModule(ident.idText, lc visitSynModuleDecl decls) + ] + | SynModuleDecl.NestedModule _ -> failwith "A nested module cannot have multiple identifiers" + | SynModuleDecl.Let (bindings = bindings) -> lc visitBinding bindings + | SynModuleDecl.Types (typeDefns = typeDefns) -> lc visitSynTypeDefn typeDefns + | SynModuleDecl.HashDirective _ -> [] + | SynModuleDecl.ModuleAbbrev _ -> failwith "no support for module abbreviations" + | SynModuleDecl.NamespaceFragment _ -> [] + | SynModuleDecl.Exception(exnDefn = SynExceptionDefn (exnRepr = SynExceptionDefnRepr (attributes = attributes + caseName = caseName + longId = longId) + members = members)) -> + [ + yield! visitSynAttributes attributes + yield! visitSynUnionCase caseName + yield! cfo visitLongIdent longId + yield! lc visitSynMemberDefn members + ] + +let visitSynModuleSigDecl (md: SynModuleSigDecl) = + match md with + | SynModuleSigDecl.Open(target = SynOpenDeclTarget.ModuleOrNamespace (longId, _)) -> + [ FileContentEntry.OpenStatement(synLongIdentToPath false longId) ] + | SynModuleSigDecl.Open(target = SynOpenDeclTarget.Type (typeName, _)) -> visitSynType typeName + | SynModuleSigDecl.NestedModule (moduleInfo = SynComponentInfo (longId = [ ident ]; attributes = attributes); moduleDecls = decls) -> + [ + yield! visitSynAttributes attributes + yield FileContentEntry.NestedModule(ident.idText, lc visitSynModuleSigDecl decls) + ] + | SynModuleSigDecl.NestedModule _ -> failwith "A nested module cannot have multiple identifiers" + | SynModuleSigDecl.ModuleAbbrev _ -> failwith "no support for module abbreviations" + | SynModuleSigDecl.Val (valSig, _) -> visitSynValSig valSig + | SynModuleSigDecl.Types (types = types) -> lc visitSynTypeDefnSig types + | SynModuleSigDecl.Exception(exnSig = SynExceptionSig (exnRepr = SynExceptionDefnRepr (attributes = attributes + caseName = caseName + longId = longId) + members = members)) -> + [ + yield! visitSynAttributes attributes + yield! visitSynUnionCase caseName + yield! cfo visitLongIdent longId + yield! lc visitSynMemberSig members + ] + | SynModuleSigDecl.HashDirective _ -> [] + | SynModuleSigDecl.NamespaceFragment _ -> [] + +let visitSynUnionCase (SynUnionCase (attributes = attributes; caseType = caseType)) = + let caseEntries = + match caseType with + | SynUnionCaseKind.Fields cases -> lc visitSynField cases + | SynUnionCaseKind.FullType (fullType = fullType) -> visitSynType fullType + + [ yield! visitSynAttributes attributes; yield! caseEntries ] + +let visitSynEnumCase (SynEnumCase (attributes = attributes)) = visitSynAttributes attributes + +let visitSynTypeDefn + (SynTypeDefn (typeInfo = SynComponentInfo (attributes = attributes; typeParams = typeParams; constraints = constraints) + typeRepr = typeRepr + members = members)) + : FileContentEntry list = + let reprEntries = + match typeRepr with + | SynTypeDefnRepr.Simple (simpleRepr, _) -> + match simpleRepr with + | SynTypeDefnSimpleRepr.Union (unionCases = unionCases) -> lc visitSynUnionCase unionCases + | SynTypeDefnSimpleRepr.Enum (cases = cases) -> lc visitSynEnumCase cases + | SynTypeDefnSimpleRepr.Record (recordFields = recordFields) -> lc visitSynField recordFields + // This is only used in the typed tree + // The parser doesn't construct this + | SynTypeDefnSimpleRepr.General _ -> [] + | SynTypeDefnSimpleRepr.LibraryOnlyILAssembly _ -> [] + | SynTypeDefnSimpleRepr.TypeAbbrev (rhsType = rhsType) -> visitSynType rhsType + | SynTypeDefnSimpleRepr.None _ -> [] + // This is only used in the typed tree + // The parser doesn't construct this + | SynTypeDefnSimpleRepr.Exception _ -> [] + | SynTypeDefnRepr.ObjectModel (kind, members, _) -> + match kind with + | SynTypeDefnKind.Delegate (signature, _) -> [ yield! visitSynType signature; yield! lc visitSynMemberDefn members ] + | _ -> lc visitSynMemberDefn members + | SynTypeDefnRepr.Exception _ -> + // This is only used in the typed tree + // The parser doesn't construct this + [] + + [ + yield! visitSynAttributes attributes + yield! cfo visitSynTyparDecls typeParams + yield! lc visitSynTypeConstraint constraints + yield! reprEntries + yield! lc visitSynMemberDefn members + ] + +let visitSynTypeDefnSig + (SynTypeDefnSig (typeInfo = SynComponentInfo (attributes = attributes; typeParams = typeParams; constraints = constraints) + typeRepr = typeRepr + members = members)) + = + let reprEntries = + match typeRepr with + | SynTypeDefnSigRepr.Simple (simpleRepr, _) -> + match simpleRepr with + | SynTypeDefnSimpleRepr.Union (unionCases = unionCases) -> lc visitSynUnionCase unionCases + | SynTypeDefnSimpleRepr.Enum (cases = cases) -> lc visitSynEnumCase cases + | SynTypeDefnSimpleRepr.Record (recordFields = recordFields) -> lc visitSynField recordFields + // This is only used in the typed tree + // The parser doesn't construct this + | SynTypeDefnSimpleRepr.General _ -> [] + | SynTypeDefnSimpleRepr.LibraryOnlyILAssembly _ -> [] + | SynTypeDefnSimpleRepr.TypeAbbrev (rhsType = rhsType) -> visitSynType rhsType + | SynTypeDefnSimpleRepr.None _ -> [] + // This is only used in the typed tree + // The parser doesn't construct this + | SynTypeDefnSimpleRepr.Exception _ -> [] + | SynTypeDefnSigRepr.ObjectModel (kind, members, _) -> + match kind with + | SynTypeDefnKind.Delegate (signature, _) -> [ yield! visitSynType signature; yield! lc visitSynMemberSig members ] + | _ -> lc visitSynMemberSig members + | SynTypeDefnSigRepr.Exception _ -> + // This is only used in the typed tree + // The parser doesn't construct this + [] + + [ + yield! visitSynAttributes attributes + yield! cfo visitSynTyparDecls typeParams + yield! lc visitSynTypeConstraint constraints + yield! reprEntries + yield! lc visitSynMemberSig members + ] + +let visitSynValSig (SynValSig (attributes = attributes; synType = synType; synExpr = synExpr)) = + [ + yield! visitSynAttributes attributes + yield! visitSynType synType + yield! cfo visitSynExpr synExpr + ] + +let visitSynField (SynField (attributes = attributes; fieldType = fieldType)) = + [ yield! visitSynAttributes attributes; yield! visitSynType fieldType ] + +let visitSynMemberDefn (md: SynMemberDefn) : FileContentEntry list = + match md with + | SynMemberDefn.Member (memberDefn = binding) -> visitBinding binding + | SynMemberDefn.Open _ -> [] + | SynMemberDefn.GetSetMember (memberDefnForGet, memberDefnForSet, _, _) -> + [ + yield! cfo visitBinding memberDefnForGet + yield! cfo visitBinding memberDefnForSet + ] + | SynMemberDefn.ImplicitCtor (ctorArgs = ctorArgs) -> visitSynSimplePats ctorArgs + | SynMemberDefn.ImplicitInherit (inheritType, inheritArgs, _, _) -> [ yield! visitSynType inheritType; yield! visitSynExpr inheritArgs ] + | SynMemberDefn.LetBindings (bindings = bindings) -> lc visitBinding bindings + | SynMemberDefn.AbstractSlot (slotSig = slotSig) -> visitSynValSig slotSig + | SynMemberDefn.Interface (interfaceType, _, members, _) -> + [ + yield! visitSynType interfaceType + yield! cfo (lc visitSynMemberDefn) members + ] + | SynMemberDefn.Inherit (baseType, _, _) -> visitSynType baseType + | SynMemberDefn.ValField (fieldInfo, _) -> visitSynField fieldInfo + | SynMemberDefn.NestedType _ -> [] + | SynMemberDefn.AutoProperty (attributes = attributes; typeOpt = typeOpt; synExpr = synExpr) -> + [ + yield! visitSynAttributes attributes + yield! cfo visitSynType typeOpt + yield! visitSynExpr synExpr + ] + +let visitSynInterfaceImpl (SynInterfaceImpl (interfaceTy = t; bindings = bindings; members = members)) = + [ + yield! visitSynType t + yield! lc visitBinding bindings + yield! lc visitSynMemberDefn members + ] + +let visitSynType (t: SynType) : FileContentEntry list = + let rec visit (t: SynType) (continuation: FileContentEntry list -> FileContentEntry list) = + match t with + | SynType.LongIdent lid -> continuation (visitSynLongIdent lid) + | SynType.App (typeName = typeName; typeArgs = typeArgs) -> + let continuations = List.map visit (typeName :: typeArgs) + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.LongIdentApp (typeName = typeName; longDotId = longDotId; typeArgs = typeArgs) -> + let continuations = List.map visit (typeName :: typeArgs) + + let finalContinuation nodes = + visitSynLongIdent longDotId @ lc id nodes |> continuation + + Continuation.sequence continuations finalContinuation + | SynType.Tuple (path = path) -> + let continuations = List.map visit (getTypeFromTuplePath path) + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.AnonRecd (fields = fields) -> + let continuations = List.map (snd >> visit) fields + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.Array (elementType = elementType) -> visit elementType continuation + | SynType.Fun (argType, returnType, _, _) -> + let continuations = List.map visit [ argType; returnType ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.Var _ -> continuation [] + | SynType.Anon _ -> continuation [] + | SynType.WithGlobalConstraints (typeName, constraints, _) -> + visit typeName (fun nodes -> nodes @ lc visitSynTypeConstraint constraints |> continuation) + | SynType.HashConstraint (innerType, _) -> visit innerType continuation + | SynType.MeasureDivide (dividend, divisor, _) -> + let continuations = List.map visit [ dividend; divisor ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.MeasurePower (baseMeasure = baseMeasure) -> visit baseMeasure continuation + | SynType.StaticConstant _ -> continuation [] + | SynType.StaticConstantExpr (expr, _) -> continuation (visitSynExpr expr) + | SynType.StaticConstantNamed (ident, value, _) -> + let continuations = List.map visit [ ident; value ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynType.Paren (innerType, _) -> visit innerType continuation + | SynType.SignatureParameter (attributes = attributes; usedType = usedType) -> + visit usedType (fun nodes -> [ yield! visitSynAttributes attributes; yield! nodes ] |> continuation) + | SynType.Or (lhsType, rhsType, _, _) -> + let continuations = List.map visit [ lhsType; rhsType ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + + visit t id + +let visitSynValTyparDecls (SynValTyparDecls (typars = typars)) = cfo visitSynTyparDecls typars + +let visitSynTyparDecls (td: SynTyparDecls) : FileContentEntry list = + match td with + | SynTyparDecls.PostfixList (decls, constraints, _) -> + [ + yield! lc visitSynTyparDecl decls + yield! lc visitSynTypeConstraint constraints + ] + | SynTyparDecls.PrefixList (decls = decls) -> lc visitSynTyparDecl decls + | SynTyparDecls.SinglePrefix (decl = decl) -> visitSynTyparDecl decl + +let visitSynTyparDecl (SynTyparDecl (attributes = attributes)) = visitSynAttributes attributes + +let visitSynTypeConstraint (tc: SynTypeConstraint) : FileContentEntry list = + match tc with + | SynTypeConstraint.WhereSelfConstrained _ -> [] + | SynTypeConstraint.WhereTyparIsValueType _ -> [] + | SynTypeConstraint.WhereTyparIsReferenceType _ -> [] + | SynTypeConstraint.WhereTyparIsUnmanaged _ -> [] + | SynTypeConstraint.WhereTyparSupportsNull _ -> [] + | SynTypeConstraint.WhereTyparIsComparable _ -> [] + | SynTypeConstraint.WhereTyparIsEquatable _ -> [] + | SynTypeConstraint.WhereTyparDefaultsToType (typeName = typeName) -> visitSynType typeName + | SynTypeConstraint.WhereTyparSubtypeOfType (typeName = typeName) -> visitSynType typeName + | SynTypeConstraint.WhereTyparSupportsMember (typars, memberSig, _) -> + [ yield! visitSynType typars; yield! visitSynMemberSig memberSig ] + | SynTypeConstraint.WhereTyparIsEnum (typeArgs = typeArgs) -> lc visitSynType typeArgs + | SynTypeConstraint.WhereTyparIsDelegate (typeArgs = typeArgs) -> lc visitSynType typeArgs + +let visitSynExpr (e: SynExpr) : FileContentEntry list = + let rec visit (e: SynExpr) (continuation: FileContentEntry list -> FileContentEntry list) : FileContentEntry list = + match e with + | SynExpr.Const _ -> continuation [] + | SynExpr.Paren (expr = expr) -> visit expr continuation + | SynExpr.Quote (operator = operator; quotedExpr = quotedExpr) -> + visit operator (fun operatorNodes -> visit quotedExpr (fun quotedNodes -> operatorNodes @ quotedNodes |> continuation)) + | SynExpr.Typed (expr, targetType, _) -> visit expr (fun nodes -> nodes @ visitSynType targetType |> continuation) + | SynExpr.Tuple (exprs = exprs) -> + let continuations = List.map visit exprs + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.AnonRecd (copyInfo = copyInfo; recordFields = recordFields) -> + let continuations = + match copyInfo with + | None -> List.map (fun (_, _, e) -> visit e) recordFields + | Some (cp, _) -> visit cp :: List.map (fun (_, _, e) -> visit e) recordFields + + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.ArrayOrList (exprs = exprs) -> + let continuations = List.map visit exprs + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.Record (baseInfo = baseInfo; copyInfo = copyInfo; recordFields = recordFields) -> + let fieldNodes = + recordFields + |> lc (fun (SynExprRecordField (fieldName = (si, _); expr = expr)) -> + [ yield! visitSynLongIdent si; yield! cfo visitSynExpr expr ]) + + match baseInfo, copyInfo with + | Some (t, e, _, _, _), None -> + visit e (fun nodes -> [ yield! visitSynType t; yield! nodes; yield! fieldNodes ] |> continuation) + | None, Some (e, _) -> visit e (fun nodes -> nodes @ fieldNodes |> continuation) + | _ -> fieldNodes + | SynExpr.New (targetType = targetType; expr = expr) -> visit expr (fun nodes -> visitSynType targetType @ nodes |> continuation) + | SynExpr.ObjExpr (objType, argOptions, _, bindings, members, extraImpls, _, _) -> + [ + yield! visitSynType objType + yield! cfo (fst >> visitSynExpr) argOptions + yield! lc visitBinding bindings + yield! lc visitSynMemberDefn members + yield! lc visitSynInterfaceImpl extraImpls + ] + |> continuation + | SynExpr.While (whileExpr = whileExpr; doExpr = doExpr) -> + visit whileExpr (fun whileNodes -> visit doExpr (fun doNodes -> whileNodes @ doNodes |> continuation)) + | SynExpr.For (identBody = identBody; toBody = toBody; doBody = doBody) -> + let continuations = List.map visit [ identBody; toBody; doBody ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.ForEach (pat = pat; enumExpr = enumExpr; bodyExpr = bodyExpr) -> + visit enumExpr (fun enumNodes -> + visit bodyExpr (fun bodyNodes -> [ yield! visitPat pat; yield! enumNodes; yield! bodyNodes ] |> continuation)) + | SynExpr.ArrayOrListComputed (expr = expr) -> visit expr continuation + | SynExpr.IndexRange (expr1 = expr1; expr2 = expr2) -> + match expr1, expr2 with + | None, None -> continuation [] + | Some e, None + | None, Some e -> visit e continuation + | Some e1, Some e2 -> visit e1 (fun e1Nodes -> visit e2 (fun e2Nodes -> e1Nodes @ e2Nodes |> continuation)) + | SynExpr.IndexFromEnd (expr, _) -> visit expr continuation + | SynExpr.ComputationExpr (expr = expr) -> visit expr continuation + | SynExpr.Lambda (args = args; body = body) -> visit body (fun bodyNodes -> visitSynSimplePats args @ bodyNodes |> continuation) + | SynExpr.MatchLambda (matchClauses = clauses) -> lc visitSynMatchClause clauses |> continuation + | SynExpr.Match (expr = expr; clauses = clauses) -> + visit expr (fun exprNodes -> [ yield! exprNodes; yield! lc visitSynMatchClause clauses ] |> continuation) + | SynExpr.Do (expr, _) -> visit expr continuation + | SynExpr.Assert (expr, _) -> visit expr continuation + | SynExpr.App (funcExpr = funcExpr; argExpr = argExpr) -> + visit funcExpr (fun funcNodes -> visit argExpr (fun argNodes -> funcNodes @ argNodes |> continuation)) + | SynExpr.TypeApp (expr = expr; typeArgs = typeArgs) -> + visit expr (fun exprNodes -> exprNodes @ lc visitSynType typeArgs |> continuation) + | SynExpr.LetOrUse (bindings = bindings; body = body) -> visit body (fun nodes -> lc visitBinding bindings @ nodes |> continuation) + | SynExpr.TryWith (tryExpr = tryExpr; withCases = withCases) -> + visit tryExpr (fun nodes -> nodes @ lc visitSynMatchClause withCases |> continuation) + | SynExpr.TryFinally (tryExpr = tryExpr; finallyExpr = finallyExpr) -> + visit tryExpr (fun tNodes -> visit finallyExpr (fun fNodes -> tNodes @ fNodes |> continuation)) + | SynExpr.Lazy (expr, _) -> visit expr continuation + | SynExpr.Sequential (expr1 = expr1; expr2 = expr2) -> + visit expr1 (fun nodes1 -> visit expr2 (fun nodes2 -> nodes1 @ nodes2 |> continuation)) + | SynExpr.IfThenElse (ifExpr = ifExpr; thenExpr = thenExpr; elseExpr = elseExpr) -> + let continuations = List.map visit (ifExpr :: thenExpr :: Option.toList elseExpr) + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.Typar _ -> continuation [] + | SynExpr.Ident _ -> continuation [] + | SynExpr.LongIdent (longDotId = longDotId) -> continuation (visitSynLongIdent longDotId) + | SynExpr.LongIdentSet (longDotId, expr, _) -> visit expr (fun nodes -> visitSynLongIdent longDotId @ nodes |> continuation) + | SynExpr.DotGet (expr = expr; longDotId = longDotId) -> + visit expr (fun nodes -> visitSynLongIdent longDotId @ nodes |> continuation) + | SynExpr.DotSet (targetExpr, longDotId, rhsExpr, _) -> + visit targetExpr (fun tNodes -> + visit rhsExpr (fun rNodes -> + [ yield! tNodes; yield! visitSynLongIdent longDotId; yield! rNodes ] + |> continuation)) + | SynExpr.Set (targetExpr, rhsExpr, _) -> + let continuations = List.map visit [ targetExpr; rhsExpr ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.DotIndexedGet (objectExpr, indexArgs, _, _) -> + let continuations = List.map visit [ objectExpr; indexArgs ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.DotIndexedSet (objectExpr, indexArgs, valueExpr, _, _, _) -> + let continuations = List.map visit [ objectExpr; indexArgs; valueExpr ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.NamedIndexedPropertySet (longDotId, expr1, expr2, _) -> + visit expr1 (fun nodes1 -> + visit expr2 (fun nodes2 -> + [ yield! visitSynLongIdent longDotId; yield! nodes1; yield! nodes2 ] + |> continuation)) + | SynExpr.DotNamedIndexedPropertySet (targetExpr, longDotId, argExpr, rhsExpr, _) -> + let continuations = List.map visit [ targetExpr; argExpr; rhsExpr ] + + let finalContinuation nodes = + visitSynLongIdent longDotId @ lc id nodes |> continuation + + Continuation.sequence continuations finalContinuation + | SynExpr.TypeTest (expr, targetType, _) -> visit expr (fun nodes -> nodes @ visitSynType targetType |> continuation) + | SynExpr.Upcast (expr, targetType, _) -> visit expr (fun nodes -> nodes @ visitSynType targetType |> continuation) + | SynExpr.Downcast (expr, targetType, _) -> visit expr (fun nodes -> nodes @ visitSynType targetType |> continuation) + | SynExpr.InferredUpcast (expr, _) -> visit expr continuation + | SynExpr.InferredDowncast (expr, _) -> visit expr continuation + | SynExpr.Null _ -> continuation [] + | SynExpr.AddressOf (expr = expr) -> visit expr continuation + | SynExpr.TraitCall (supportTys, traitSig, argExpr, _) -> + visit argExpr (fun nodes -> + [ + yield! visitSynType supportTys + yield! visitSynMemberSig traitSig + yield! nodes + ] + |> continuation) + | SynExpr.JoinIn (lhsExpr, _, rhsExpr, _) -> + let continuations = List.map visit [ lhsExpr; rhsExpr ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.ImplicitZero _ -> continuation [] + | SynExpr.SequentialOrImplicitYield (_, expr1, expr2, _, _) -> + let continuations = List.map visit [ expr1; expr2 ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.YieldOrReturn (expr = expr) -> visit expr continuation + | SynExpr.YieldOrReturnFrom (expr = expr) -> visit expr continuation + | SynExpr.LetOrUseBang (pat = pat; rhs = rhs; andBangs = andBangs; body = body) -> + let continuations = + let andBangExprs = List.map (fun (SynExprAndBang (body = body)) -> body) andBangs + List.map visit (body :: rhs :: andBangExprs) + + let finalContinuation nodes = + [ + yield! lc id nodes + yield! visitPat pat + yield! lc (fun (SynExprAndBang (pat = pat)) -> visitPat pat) andBangs + ] + |> continuation + + Continuation.sequence continuations finalContinuation + | SynExpr.MatchBang (expr = expr; clauses = clauses) -> + visit expr (fun exprNodes -> [ yield! exprNodes; yield! lc visitSynMatchClause clauses ] |> continuation) + | SynExpr.DoBang (expr, _) -> visit expr continuation + | SynExpr.LibraryOnlyILAssembly (typeArgs = typeArgs; args = args; retTy = retTy) -> + let typeNodes = lc visitSynType (typeArgs @ retTy) + let continuations = List.map visit args + let finalContinuation nodes = lc id nodes @ typeNodes |> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.LibraryOnlyStaticOptimization (constraints, expr, optimizedExpr, _) -> + let constraintTypes = + constraints + |> List.choose (function + | SynStaticOptimizationConstraint.WhenTyparTyconEqualsTycon (rhsType = t) -> Some t + | SynStaticOptimizationConstraint.WhenTyparIsStruct _ -> None) + + visit expr (fun eNodes -> + visit optimizedExpr (fun oNodes -> + [ yield! lc visitSynType constraintTypes; yield! eNodes; yield! oNodes ] + |> continuation)) + | SynExpr.LibraryOnlyUnionCaseFieldGet (expr, longId, _, _) -> + visit expr (fun eNodes -> visitLongIdent longId @ eNodes |> continuation) + | SynExpr.LibraryOnlyUnionCaseFieldSet (expr, longId, _, rhsExpr, _) -> + visit expr (fun eNodes -> + visit rhsExpr (fun rhsNodes -> [ yield! visitLongIdent longId; yield! eNodes; yield! rhsNodes ] |> continuation)) + | SynExpr.ArbitraryAfterError _ -> continuation [] + | SynExpr.FromParseError _ -> continuation [] + | SynExpr.DiscardAfterMissingQualificationAfterDot _ -> continuation [] + | SynExpr.Fixed (expr, _) -> visit expr continuation + | SynExpr.InterpolatedString (contents = contents) -> + let continuations = + List.map + visit + (List.choose + (function + | SynInterpolatedStringPart.FillExpr (fillExpr = e) -> Some e + | SynInterpolatedStringPart.String _ -> None) + contents) + + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynExpr.DebugPoint _ -> continuation [] + | SynExpr.Dynamic (funcExpr, _, argExpr, _) -> + let continuations = List.map visit [ funcExpr; argExpr ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + + visit e id + +let visitPat (p: SynPat) : FileContentEntry list = + let rec visit (p: SynPat) (continuation: FileContentEntry list -> FileContentEntry list) : FileContentEntry list = + match p with + | SynPat.Paren (pat = pat) -> visit pat continuation + | SynPat.Typed (pat = pat; targetType = t) -> visit pat (fun nodes -> nodes @ visitSynType t) + | SynPat.Const _ -> continuation [] + | SynPat.Wild _ -> continuation [] + | SynPat.Named _ -> continuation [] + | SynPat.Attrib (pat, attributes, _) -> visit pat (fun nodes -> visitSynAttributes attributes @ nodes |> continuation) + | SynPat.Or (lhsPat, rhsPat, _, _) -> + let continuations = List.map visit [ lhsPat; rhsPat ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.ListCons (lhsPat, rhsPat, _, _) -> + let continuations = List.map visit [ lhsPat; rhsPat ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.Ands (pats, _) -> + let continuations = List.map visit pats + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.As (lhsPat, rhsPat, _) -> + let continuations = List.map visit [ lhsPat; rhsPat ] + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.LongIdent (longDotId = longDotId; typarDecls = typarDecls; argPats = argPats) -> + continuation + [ + yield! visitSynLongIdent longDotId + yield! cfo visitSynValTyparDecls typarDecls + yield! visitSynArgPats argPats + ] + | SynPat.Tuple (_, elementPats, _) -> + let continuations = List.map visit elementPats + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.ArrayOrList (_, elementPats, _) -> + let continuations = List.map visit elementPats + let finalContinuation = lc id >> continuation + Continuation.sequence continuations finalContinuation + | SynPat.Record (fieldPats, _) -> + let pats = List.map (fun (_, _, p) -> p) fieldPats + let lids = lc (fun ((l, _), _, _) -> visitLongIdent l) fieldPats + let continuations = List.map visit pats + + let finalContinuation nodes = + [ yield! lc id nodes; yield! lids ] |> continuation + + Continuation.sequence continuations finalContinuation + | SynPat.Null _ -> continuation [] + | SynPat.OptionalVal _ -> continuation [] + | SynPat.IsInst (t, _) -> continuation (visitSynType t) + | SynPat.QuoteExpr (expr, _) -> continuation (visitSynExpr expr) + | SynPat.DeprecatedCharRange _ -> continuation [] + | SynPat.InstanceMember _ -> continuation [] + | SynPat.FromParseError _ -> continuation [] + + visit p id + +let visitSynArgPats (argPat: SynArgPats) = + match argPat with + | SynArgPats.Pats args -> lc visitPat args + | SynArgPats.NamePatPairs (pats = pats) -> lc (fun (_, _, p) -> visitPat p) pats + +let visitSynSimplePat (pat: SynSimplePat) = + match pat with + | SynSimplePat.Id _ -> [] + | SynSimplePat.Attrib (pat, attributes, _) -> [ yield! visitSynSimplePat pat; yield! visitSynAttributes attributes ] + | SynSimplePat.Typed (pat, t, _) -> [ yield! visitSynSimplePat pat; yield! visitSynType t ] + +let visitSynSimplePats (pats: SynSimplePats) = + match pats with + | SynSimplePats.SimplePats (pats = pats) -> lc visitSynSimplePat pats + | SynSimplePats.Typed (pats, t, _) -> [ yield! visitSynSimplePats pats; yield! visitSynType t ] + +let visitSynMatchClause (SynMatchClause (pat = pat; whenExpr = whenExpr; resultExpr = resultExpr)) = + [ + yield! visitPat pat + yield! cfo visitSynExpr whenExpr + yield! visitSynExpr resultExpr + ] + +let visitBinding (SynBinding (attributes = attributes; headPat = headPat; returnInfo = returnInfo; expr = expr)) : FileContentEntry list = + let pattern = + match headPat with + | SynPat.LongIdent(argPats = SynArgPats.Pats pats) -> lc visitPat pats + | _ -> visitPat headPat + + [ + yield! visitSynAttributes attributes + yield! pattern + yield! cfo visitSynBindingReturnInfo returnInfo + yield! visitSynExpr expr + ] + +let visitSynBindingReturnInfo (SynBindingReturnInfo (typeName = typeName; attributes = attributes)) = + [ yield! visitSynAttributes attributes; yield! visitSynType typeName ] + +let visitSynMemberSig (ms: SynMemberSig) : FileContentEntry list = + match ms with + | SynMemberSig.Member (memberSig = memberSig) -> visitSynValSig memberSig + | SynMemberSig.Interface (interfaceType, _) -> visitSynType interfaceType + | SynMemberSig.Inherit (inheritedType, _) -> visitSynType inheritedType + | SynMemberSig.ValField (field, _) -> visitSynField field + | SynMemberSig.NestedType _ -> [] + +let mkFileContent (f: FileWithAST) : FileContentEntry list = + match f.AST with + | ParsedInput.SigFile (ParsedSigFileInput (contents = contents)) -> + lc + (fun (SynModuleOrNamespaceSig (longId = longId; kind = kind; decls = decls; attribs = attribs)) -> + let attributes = lc visitSynAttributeList attribs + + let contentEntries = + match kind with + | SynModuleOrNamespaceKind.GlobalNamespace + | SynModuleOrNamespaceKind.AnonModule -> lc visitSynModuleSigDecl decls + | SynModuleOrNamespaceKind.DeclaredNamespace -> + let path = longIdentToPath false longId + + [ FileContentEntry.TopLevelNamespace(path, lc visitSynModuleSigDecl decls) ] + | SynModuleOrNamespaceKind.NamedModule -> + let path = longIdentToPath true longId + + [ FileContentEntry.TopLevelNamespace(path, lc visitSynModuleSigDecl decls) ] + + [ yield! attributes; yield! contentEntries ]) + contents + + | ParsedInput.ImplFile (ParsedImplFileInput (contents = contents)) -> + lc + (fun (SynModuleOrNamespace (longId = longId; attribs = attribs; kind = kind; decls = decls)) -> + let attributes = lc visitSynAttributeList attribs + + let contentEntries = + match kind with + | SynModuleOrNamespaceKind.GlobalNamespace + | SynModuleOrNamespaceKind.AnonModule -> lc visitSynModuleDecl decls + | SynModuleOrNamespaceKind.DeclaredNamespace -> + let path = longIdentToPath false longId + + [ FileContentEntry.TopLevelNamespace(path, lc visitSynModuleDecl decls) ] + | SynModuleOrNamespaceKind.NamedModule -> + let path = longIdentToPath true longId + + [ FileContentEntry.TopLevelNamespace(path, lc visitSynModuleDecl decls) ] + + [ yield! attributes; yield! contentEntries ]) + contents + +// ================================================================================================================================ +// ================================================================================================================================ +module Tests = + open NUnit.Framework + open FSharp.Compiler.Service.Tests.Common + + [] + let ``Test a single file`` () = + let fileName = + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fsi" + + let ast = parseSourceCode (fileName, System.IO.File.ReadAllText(fileName)) + let contents = mkFileContent { Idx = 0; File = fileName; AST = ast } + ignore contents diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs similarity index 80% rename from tests/ParallelTypeCheckingTests/Code/TrieApproach.fs rename to tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs index a3ae7d4e3e..e598e337b7 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs @@ -1,150 +1,27 @@ -module ParallelTypeCheckingTests.Code.TrieApproach +module ParallelTypeCheckingTests.Code.TrieApproach.SampleData open System.Collections.Generic open NUnit.Framework +open ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution -// This is pseudo code of how we could restructure the trie code -// My main benefit is that you can easily visually inspect if an identifier will match something in the trie +let dictionary<'key, 'value when 'key: equality> (entries: ('key * 'value) seq) = + entries |> Seq.map (fun (k, v) -> KeyValuePair(k, v)) |> Dictionary -type File = string -type Files = Set -type ModuleSegment = string - -/// There is a subtle difference a module and namespace. -/// A namespace does not necessarily expose a set of dependent files. -/// Only when the namespace exposes types that could later be inferred. -/// Children of a namespace don't automatically depend on each other for that reason -type TrieNodeInfo = - | Root - | Module of segment: string * file: File - | Namespace of segment: string * filesThatExposeTypes: Files - - member x.Segment = - match x with - | Root -> failwith "Root has no segment" - | Module (segment = segment) - | Namespace (segment = segment) -> segment - - member x.Files: Files = - match x with - | Root -> failwith "Root has no files" - | Module (file = file) -> Set.singleton file - | Namespace (filesThatExposeTypes = files) -> set files - -type TrieNode = - { - Current: TrieNodeInfo - Children: IDictionary - } - - member x.Files = x.Current.Files - -type FileContentEntry = - /// Any toplevel namespace a file might have. - /// In case a file has `module X.Y.Z`, then `X.Y` is considered to be the toplevel namespace - | TopLevelNamespace of path: ModuleSegment list * content: FileContentEntry list - /// The `open X.Y.Z` syntax. - | OpenStatement of path: ModuleSegment list - /// Any identifier that has more than one piece (LongIdent or SynLongIdent) in it. - /// The last part of the identifier should not be included. - | PrefixedIdentifier of path: ModuleSegment list - /// Being explicit about nested modules allows for easier reasoning what namespaces (paths) are open. - /// We can scope an `OpenStatement` to the everything that is happening inside the nested module. - | NestedModule of name: string * nestedContent: FileContentEntry list - -type FileContent = - { - Name: File - Idx: int - Content: FileContentEntry array - } - -type FileContentQueryState = - { - OpenNamespaces: Set - FoundDependencies: Set - CurrentFile: File - KnownFiles: Files - } - - static member Create (file: File) (knownFiles: Files) = - { - OpenNamespaces = Set.empty - FoundDependencies = Set.empty - CurrentFile = file - KnownFiles = knownFiles - } - - member x.AddDependencies(files: Files) : FileContentQueryState = - let files = Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies - { x with FoundDependencies = files } - - member x.AddOpenNamespace(path: ModuleSegment list) = - { x with - OpenNamespaces = Set.add path x.OpenNamespaces - } - - member x.AddDependenciesAndOpenNamespace(files: Files, path: ModuleSegment list) = - let foundDependencies = - Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies - - { x with - FoundDependencies = foundDependencies - OpenNamespaces = Set.add path x.OpenNamespaces - } - -[] -type QueryTrieNodeResult = - /// No node was found for the path in the trie - | NodeDoesNotExist - /// A node was found but it yielded no file links - | NodeDoesNotExposeData - /// A node was found with one or more file links - | NodeExposesData of Files - -// This code just looks for a path in the trie -// It could be cached and is easy to reason about. -let queryTrie (trie: TrieNode) (path: ModuleSegment list) : QueryTrieNodeResult = - let rec visit (currentNode: TrieNode) (path: ModuleSegment list) = - match path with - | [] -> failwith "path should not be empty" - | [ lastNodeFromPath ] -> - let childResults = - currentNode.Children - |> Seq.tryFind (fun (KeyValue (segment, _childNode)) -> segment = lastNodeFromPath) - - match childResults with - | None -> QueryTrieNodeResult.NodeDoesNotExist - | Some (KeyValue (_, childNode)) -> - if Set.isEmpty childNode.Files then - QueryTrieNodeResult.NodeDoesNotExposeData - else - QueryTrieNodeResult.NodeExposesData(childNode.Files) - | currentPath :: restPath -> - let childResults = - currentNode.Children - |> Seq.tryFind (fun (KeyValue (segment, _childNode)) -> segment = currentPath) - - match childResults with - | None -> QueryTrieNodeResult.NodeDoesNotExist - | Some (KeyValue (_, childNode)) -> visit childNode restPath - - visit trie path - -let noChildren = dict [||] +let noChildren = Dictionary(0) +let emptyHS () = HashSet(0) // This should be constructed from the AST let fantomasCoreTrie: TrieNode = { Current = TrieNodeInfo.Root Children = - dict + dictionary [| "System", { - Current = TrieNodeInfo.Namespace("System", Set.empty) + Current = TrieNodeInfo.Namespace("System", emptyHS ()) Children = - dict + dictionary [| "AssemblyVersionInformation", { @@ -155,15 +32,15 @@ let fantomasCoreTrie: TrieNode = } "Fantomas", { - Current = TrieNodeInfo.Namespace("Fantomas", Set.empty) + Current = TrieNodeInfo.Namespace("Fantomas", emptyHS ()) Children = - dict + dictionary [| "Core", { - Current = TrieNodeInfo.Namespace("Core", Set.empty) + Current = TrieNodeInfo.Namespace("Core", emptyHS ()) Children = - dict + dictionary [| "ISourceTextExtensions", { @@ -182,7 +59,7 @@ let fantomasCoreTrie: TrieNode = } "AstExtensions", { - Current = TrieNodeInfo.Module("AstExtensions", "AstExtensions.fs") + Current = TrieNodeInfo.Module("AstExtensions", "AstExtensions.fsi") Children = noChildren } "TriviaTypes", @@ -844,122 +721,23 @@ let files = } |] -// Now how to detect the deps between files? -// Process the content of each file using some state - -// Helper function to process a open statement -// The statement could link to files and/or should be tracked as an open namespace -let processOpenPath (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = - let queryResult = queryTrie fantomasCoreTrie path - - match queryResult with - | QueryTrieNodeResult.NodeDoesNotExist -> state - | QueryTrieNodeResult.NodeDoesNotExposeData -> state.AddOpenNamespace path - | QueryTrieNodeResult.NodeExposesData files -> state.AddDependenciesAndOpenNamespace(files, path) - -// Helper function to process an identifier -let processIdentifier (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = - let queryResult = queryTrie fantomasCoreTrie path - - match queryResult with - | QueryTrieNodeResult.NodeDoesNotExist -> state - | QueryTrieNodeResult.NodeDoesNotExposeData -> failwith "This identifier cannot be part of a node that doesn't expose data!" - | QueryTrieNodeResult.NodeExposesData files -> state.AddDependencies files - -// Typically used to folder FileContentEntry items over a FileContentQueryState -let rec processStateEntry (state: FileContentQueryState) (entry: FileContentEntry) : FileContentQueryState = - match entry with - | FileContentEntry.TopLevelNamespace (topLevelPath, content) -> - let state = - match topLevelPath with - | [] -> state - | _ -> state.AddOpenNamespace topLevelPath - - List.fold processStateEntry state content - - | FileContentEntry.OpenStatement path -> - // An open statement can directly reference file or be a partial open statement - // Both cases need to be processed. - let stateAfterFullOpenPath = processOpenPath path state - - // Any existing open statement could be extended with the current path (if that node where to exists in the trie) - // The extended path could add a new link (in case of a module or namespace with types) - // It might also not add anything at all (in case it the extended path is still a partial one) - (stateAfterFullOpenPath, state.OpenNamespaces) - ||> Seq.fold (fun acc openNS -> processOpenPath [ yield! openNS; yield! path ] acc) - - | FileContentEntry.PrefixedIdentifier path -> - // process the name was if it were a FQN - let stateAfterFullIdentifier = processIdentifier path state - - // Process the name in combination with the existing open namespaces - (stateAfterFullIdentifier, state.OpenNamespaces) - ||> Seq.fold (fun acc openNS -> processIdentifier [ yield! openNS; yield! path ] acc) - - | FileContentEntry.NestedModule (nestedContent = nestedContent) -> - // We don't want our current state to be affect by any open statements in the nested module - let nestedState = List.fold processStateEntry state nestedContent - // Afterward we are only interested in the found dependencies in the nested module - let foundDependencies = - Set.union state.FoundDependencies nestedState.FoundDependencies - - { state with - FoundDependencies = foundDependencies - } - -let getFileNameBefore idx = - files.[0 .. (idx - 1)] |> Array.map (fun f -> f.Name) |> Set.ofArray - -let mkGraph files = - files - |> Array.map (fun (file: FileContent) -> - let knownFiles = getFileNameBefore file.Idx - - let result = - Array.fold processStateEntry (FileContentQueryState.Create file.Name knownFiles) file.Content - - file.Name, Set.toArray result.FoundDependencies) - [] let ``Full project simulation`` () = - let graph = mkGraph files + let graph = + files + |> Array.map (fun fileContent -> + let knownFiles = + files.[0 .. (fileContent.Idx - 1)] |> Array.map (fun f -> f.Name) |> set + + let result = + Seq.fold (processStateEntry fantomasCoreTrie) (FileContentQueryState.Create fileContent.Name knownFiles) fileContent.Content + + fileContent.Name, Set.toArray result.FoundDependencies) for fileName, deps in graph do let depString = String.concat ", " deps printfn $"%s{fileName}: [{depString}]" - () - -[] -let ``SourceParser.fs simulation`` () = - let fileName = "SourceParser.fs" - let file = Array.find (fun (fc: FileContent) -> fc.Name = fileName) files - let knownFiles = getFileNameBefore file.Idx - - let result = - Array.fold processStateEntry (FileContentQueryState.Create file.Name knownFiles) file.Content - - let deps = Seq.sort result.FoundDependencies |> Seq.toList - - match deps with - | [ "AstExtensions.fs"; "RangeHelpers.fs"; "TriviaTypes.fs"; "Utils.fs" ] -> Assert.Pass() - | deps -> Assert.Fail $"Unexpected deps for {fileName}, got %A{deps}" - -[] -let ``AstExtensions.fs simulation`` () = - let fileName = "AstExtensions.fs" - let file = Array.find (fun (fc: FileContent) -> fc.Name = fileName) files - let knownFiles = getFileNameBefore file.Idx - - let result = - Array.fold processStateEntry (FileContentQueryState.Create file.Name knownFiles) file.Content - - let deps = Seq.sort result.FoundDependencies |> Seq.toList - - match deps with - | [ "RangeHelpers.fs" ] -> Assert.Pass() - | deps -> Assert.Fail $"Unexpected deps for {fileName}, got %A{deps}" - [] let ``Query non existing node in trie`` () = let result = @@ -994,11 +772,23 @@ let ``ProcessOpenStatement full path match`` () = Array.find (fun (f: FileContent) -> f.Name = "SourceParser.fs") files let state = - FileContentQueryState.Create sourceParser.Name (getFileNameBefore sourceParser.Idx) + FileContentQueryState.Create + sourceParser.Name + (set + [| + "AssemblyInfo.fs" + "ISourceTextExtensions.fs" + "RangeHelpers.fs" + "AstExtensions.fsi" + "TriviaTypes.fs" + "Utils.fs" + |]) + + let result = + processOpenPath fantomasCoreTrie [ "Fantomas"; "Core"; "AstExtensions" ] state - let result = processOpenPath [ "Fantomas"; "Core"; "AstExtensions" ] state let dep = Seq.exactlyOne result.FoundDependencies - Assert.AreEqual("AstExtensions.fs", dep) + Assert.AreEqual("AstExtensions.fsi", dep) #if INTERACTIVE open System.Text.RegularExpressions diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs new file mode 100644 index 0000000000..4c772a90fb --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs @@ -0,0 +1,271 @@ +module ParallelTypeCheckingTests.Code.TrieApproach.TrieMapping + +open System.Collections.Generic +open System.Diagnostics +open FSharp.Compiler.Syntax +open Microsoft.FSharp.Collections + +let time msg f a = + let sw = Stopwatch.StartNew() + let result = f a + sw.Stop() + printfn $"{msg} took %A{sw.Elapsed}" + result + +let mergeTrieNodes (defaultChildSize: int) (tries: TrieNode seq) = + let rec mergeTrieNodesAux (root: TrieNode) (KeyValue (k, v)) = + if root.Children.ContainsKey k then + let node = root.Children.[k] + + match node.Current, v.Current with + | TrieNodeInfo.Namespace (filesThatExposeTypes = currentFiles), TrieNodeInfo.Namespace (filesThatExposeTypes = otherFiles) -> + for otherFile in otherFiles do + do () + currentFiles.Add(otherFile) |> ignore + | _ -> () + + for kv in v.Children do + mergeTrieNodesAux node kv + + else + root.Children.Add(k, v) + + match Seq.tryExactlyOne tries with + | Some singleTrie -> + assert (singleTrie.Current = TrieNodeInfo.Root) + singleTrie + | None -> + let root = + { + Current = TrieNodeInfo.Root + Children = Dictionary<_, _>(defaultChildSize) + } + + do () + + for trie in tries do + assert (trie.Current = TrieNodeInfo.Root) + + for kv in trie.Children do + mergeTrieNodesAux root kv + + root + +let hs f = HashSet(Seq.singleton f) +let emptyHS () = HashSet(0) + +let rec mkTrieNodeFor (file: FileWithAST) : TrieNode = + match file.AST with + | ParsedInput.SigFile (ParsedSigFileInput (contents = contents)) -> + contents + |> List.choose (fun (SynModuleOrNamespaceSig (longId = longId; kind = kind; decls = decls; accessibility = _accessibility)) -> + let hasTypes = + List.exists + (function + | SynModuleSigDecl.Types _ -> true + | _ -> false) + decls + + let isNamespace = + match kind with + | SynModuleOrNamespaceKind.AnonModule + | SynModuleOrNamespaceKind.NamedModule -> false + | SynModuleOrNamespaceKind.DeclaredNamespace -> true + | SynModuleOrNamespaceKind.GlobalNamespace -> failwith "Not quite sure yet how to perceive this" + + match longId with + | [] -> None + | _ -> + let rootNode = + let rec visit continuation (xs: LongIdent) = + match xs with + | [] -> failwith "should even empty" + | [ finalPart ] -> + let name = finalPart.idText + + let current = + if isNamespace then + TrieNodeInfo.Namespace(name, (if hasTypes then hs file.File else emptyHS ())) + else + TrieNodeInfo.Module(name, file.File) + + let children = List.choose (mkTrieForNestedSigModule file.File) decls + + continuation ( + Dictionary<_, _>( + Seq.singleton ( + KeyValuePair( + name, + { + Current = current + Children = Dictionary(children) + } + ) + ) + ) + ) + | head :: tail -> + let name = head.idText + + visit + (fun node -> + let current = TrieNodeInfo.Namespace(name, emptyHS ()) + + Dictionary<_, _>(Seq.singleton (KeyValuePair(name, { Current = current; Children = node }))) + |> continuation) + tail + + visit id longId + + Some { Current = Root; Children = rootNode }) + |> mergeTrieNodes contents.Length + | ParsedInput.ImplFile (ParsedImplFileInput (contents = contents)) -> + contents + |> List.choose (fun (SynModuleOrNamespace (longId = longId; kind = kind; decls = decls; accessibility = _accessibility)) -> + let hasTypes = + List.exists + (function + | SynModuleDecl.Types _ -> true + | _ -> false) + decls + + let isNamespace = + match kind with + | SynModuleOrNamespaceKind.AnonModule + | SynModuleOrNamespaceKind.NamedModule -> false + | SynModuleOrNamespaceKind.DeclaredNamespace -> true + | SynModuleOrNamespaceKind.GlobalNamespace -> failwith "Not quite sure yet how to perceive this" + + match longId with + | [] -> None + | _ -> + let rootNode = + let rec visit continuation (xs: LongIdent) = + match xs with + | [] -> failwith "should even empty" + | [ finalPart ] -> + let name = finalPart.idText + + let current = + if isNamespace then + TrieNodeInfo.Namespace(name, (if hasTypes then hs file.File else emptyHS ())) + else + TrieNodeInfo.Module(name, file.File) + + let children = List.choose (mkTrieForNestedModule file.File) decls + + continuation ( + Dictionary<_, _>( + Seq.singleton ( + KeyValuePair( + name, + { + Current = current + Children = Dictionary(children) + } + ) + ) + ) + ) + | head :: tail -> + let name = head.idText + + visit + (fun node -> + let current = TrieNodeInfo.Namespace(name, emptyHS ()) + + Dictionary<_, _>(Seq.singleton (KeyValuePair(name, { Current = current; Children = node }))) + |> continuation) + tail + + visit id longId + + Some { Current = Root; Children = rootNode }) + |> mergeTrieNodes contents.Length + +and mkTrieForNestedModule file (decl: SynModuleDecl) : KeyValuePair option = + match decl with + | SynModuleDecl.NestedModule (moduleInfo = SynComponentInfo(longId = [ nestedModuleIdent ]); decls = decls) -> + let name = nestedModuleIdent.idText + let children = List.choose (mkTrieForNestedModule file) decls + + Some( + KeyValuePair( + name, + { + Current = TrieNodeInfo.Module(name, file) + Children = Dictionary(children) + } + ) + ) + + | _ -> None + +and mkTrieForNestedSigModule file (decl: SynModuleSigDecl) : KeyValuePair option = + match decl with + | SynModuleSigDecl.NestedModule (moduleInfo = SynComponentInfo(longId = [ nestedModuleIdent ]); moduleDecls = decls) -> + let name = nestedModuleIdent.idText + let children = List.choose (mkTrieForNestedSigModule file) decls + + Some( + KeyValuePair( + name, + { + Current = TrieNodeInfo.Module(name, file) + Children = Dictionary(children) + } + ) + ) + + | _ -> None + +let mkTrie (files: FileWithAST array) : TrieNode = + let tries = Array.Parallel.map mkTrieNodeFor files + time "mergeTrieNodes" (mergeTrieNodes 0) tries + +// ================================================================================================================================================== +// ================================================================================================================================================== +open FSharp.Compiler.Service.Tests.Common +open NUnit.Framework + +[] +let ``Fantomas Core trie`` () = + let files = + [| + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\Fantomas.Core.AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\ISourceTextExtensions.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\RangeHelpers.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fsi" + // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\TriviaTypes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Utils.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceParser.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fsi" + // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Version.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Queue.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\FormatConfig.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fsi" + // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceTransformer.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Context.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fsi" + // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fsi" + // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Validation.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fsi" + // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fsi" + // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fs" + |] + |> Array.mapi (fun idx file -> + let ast = parseSourceCode (file, System.IO.File.ReadAllText(file)) + { Idx = idx; File = file; AST = ast }) + + let trie = mkTrie files + ignore trie diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs new file mode 100644 index 0000000000..f7bccabcff --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs @@ -0,0 +1,107 @@ +namespace ParallelTypeCheckingTests.Code.TrieApproach + +open System.Collections.Generic +open FSharp.Compiler.Syntax + +type File = string +type Files = Set +type ModuleSegment = string + +type FileWithAST = + { + Idx: int + File: File + AST: ParsedInput + } + +/// There is a subtle difference a module and namespace. +/// A namespace does not necessarily expose a set of dependent files. +/// Only when the namespace exposes types that could later be inferred. +/// Children of a namespace don't automatically depend on each other for that reason +type TrieNodeInfo = + | Root + | Module of segment: string * file: File + | Namespace of segment: string * filesThatExposeTypes: HashSet + + member x.Segment = + match x with + | Root -> failwith "Root has no segment" + | Module (segment = segment) + | Namespace (segment = segment) -> segment + + member x.Files: Files = + match x with + | Root -> failwith "Root has no files" + | Module (file = file) -> Set.singleton file + | Namespace (filesThatExposeTypes = files) -> set files + +type TrieNode = + { + Current: TrieNodeInfo + Children: Dictionary + } + + member x.Files = x.Current.Files + +type FileContentEntry = + /// Any toplevel namespace a file might have. + /// In case a file has `module X.Y.Z`, then `X.Y` is considered to be the toplevel namespace + | TopLevelNamespace of path: ModuleSegment list * content: FileContentEntry list + /// The `open X.Y.Z` syntax. + | OpenStatement of path: ModuleSegment list + /// Any identifier that has more than one piece (LongIdent or SynLongIdent) in it. + /// The last part of the identifier should not be included. + | PrefixedIdentifier of path: ModuleSegment list + /// Being explicit about nested modules allows for easier reasoning what namespaces (paths) are open. + /// We can scope an `OpenStatement` to the everything that is happening inside the nested module. + | NestedModule of name: string * nestedContent: FileContentEntry list + +type FileContent = + { + Name: File + Idx: int + Content: FileContentEntry array + } + +type FileContentQueryState = + { + OpenNamespaces: Set + FoundDependencies: Set + CurrentFile: File + KnownFiles: Files + } + + static member Create (file: File) (knownFiles: Files) = + { + OpenNamespaces = Set.empty + FoundDependencies = Set.empty + CurrentFile = file + KnownFiles = knownFiles + } + + member x.AddDependencies(files: Files) : FileContentQueryState = + let files = Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies + { x with FoundDependencies = files } + + member x.AddOpenNamespace(path: ModuleSegment list) = + { x with + OpenNamespaces = Set.add path x.OpenNamespaces + } + + member x.AddDependenciesAndOpenNamespace(files: Files, path: ModuleSegment list) = + let foundDependencies = + Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies + + { x with + FoundDependencies = foundDependencies + OpenNamespaces = Set.add path x.OpenNamespaces + } + +[] +type QueryTrieNodeResult = + /// No node was found for the path in the trie + | NodeDoesNotExist + /// A node was found but it yielded no file links + | NodeDoesNotExposeData + /// A node was found with one or more file links + | NodeExposesData of Files diff --git a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj index 2990980afa..fba8557db3 100644 --- a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj +++ b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj @@ -37,7 +37,11 @@ - + + + + + From 7dc79af0e3bf2a6daf46215ce45ef143ef80dc13 Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 17 Nov 2022 13:53:15 +0100 Subject: [PATCH 03/21] Clean up --- .../Code/TrieApproach/FileContentMapping.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs index d2b33de572..064f476697 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs @@ -12,17 +12,17 @@ module Continuation = type Continuations = ((FileContentEntry list -> FileContentEntry list) -> FileContentEntry list) list +/// Option.toList >> (List.collect f) let cfo f a = lc f (Option.toList a) +/// List.collect let lc = List.collect -let identToPath (ident: Ident) = ident.idText - let longIdentToPath (skipLast: bool) (longId: LongIdent) : ModuleSegment list = if skipLast then List.take (longId.Length - 1) longId else longId - |> List.map identToPath + |> List.map (fun ident -> ident.idText) let synLongIdentToPath (skipLast: bool) (synLongIdent: SynLongIdent) = longIdentToPath skipLast synLongIdent.LongIdent From 029b0f67648192dbe27d2d8b2f4b86035fd75e2f Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 17 Nov 2022 16:12:46 +0100 Subject: [PATCH 04/21] Address SynModuleDecl.ModuleAbbrev. Verify locally with FCS project. --- .../Code/TrieApproach/DependencyResolution.fs | 428 ++++++++++++++++-- .../Code/TrieApproach/FileContentMapping.fs | 5 +- .../Code/TrieApproach/TrieMapping.fs | 19 +- .../ParallelTypeCheckingTests.fsproj | 4 +- 4 files changed, 396 insertions(+), 60 deletions(-) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs index 8acb6c2256..bc0d5b8224 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs @@ -103,6 +103,13 @@ let rec processStateEntry (trie: TrieNode) (state: FileContentQueryState) (entry let getFileNameBefore (files: FileWithAST array) idx = files.[0 .. (idx - 1)] |> Array.map (fun f -> f.File) |> Set.ofArray +let time msg f a = + let sw = System.Diagnostics.Stopwatch.StartNew() + let result = f a + sw.Stop() + printfn $"{msg} took %A{sw.Elapsed}" + result + let mkGraph (files: FileWithAST array) = let trie = let input = @@ -112,19 +119,22 @@ let mkGraph (files: FileWithAST array) = | ParsedInput.SigFile _ -> true | ParsedInput.ImplFile _ -> Array.forall (fun (sigFile: FileWithAST) -> sigFile.File <> $"{f.File}i") files) - TrieMapping.mkTrie input + time "TrieMapping.mkTrie" TrieMapping.mkTrie input let fileContents = Array.Parallel.map FileContentMapping.mkFileContent files - files - |> Array.map (fun (file: FileWithAST) -> - let fileContent = fileContents.[file.Idx] - let knownFiles = getFileNameBefore files file.Idx + time + "mkGraph" + Array.Parallel.map + (fun (file: FileWithAST) -> + let fileContent = fileContents.[file.Idx] + let knownFiles = getFileNameBefore files file.Idx - let result = - Seq.fold (processStateEntry trie) (FileContentQueryState.Create file.File knownFiles) fileContent + let result = + Seq.fold (processStateEntry trie) (FileContentQueryState.Create file.File knownFiles) fileContent - file.File, Set.toArray result.FoundDependencies) + file.File, Set.toArray result.FoundDependencies) + files // ============================================================================================================= // ============================================================================================================= @@ -132,42 +142,7 @@ let mkGraph (files: FileWithAST array) = open NUnit.Framework open FSharp.Compiler.Service.Tests.Common -[] -let ``Fantomas.Core for realzies`` () = - let files = - [| - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\Fantomas.Core.AssemblyInfo.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AssemblyInfo.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\ISourceTextExtensions.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\RangeHelpers.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fsi" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\TriviaTypes.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Utils.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceParser.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fsi" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Version.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Queue.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\FormatConfig.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fsi" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fsi" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceTransformer.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Context.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fsi" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fsi" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Validation.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fsi" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fs" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fsi" - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fs" - |] - +let mkGraphAndReport files = let filesWithAST = files |> Array.mapi (fun idx file -> @@ -186,3 +161,368 @@ let ``Fantomas.Core for realzies`` () = printfn $"%s{fileName}: []" else printfn $"%s{fileName}:\n {depString}" + +[] +let ``Fantomas.Core for realzies`` () = + [| + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\obj\Debug\netstandard2.0\Fantomas.Core.AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\ISourceTextExtensions.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\RangeHelpers.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\TriviaTypes.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Utils.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceParser.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Version.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Queue.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\FormatConfig.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceTransformer.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Context.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Validation.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fs" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fs" + |] + |> mkGraphAndReport + +let fcsFiles = + [| + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSComp.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSIstrings.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\UtilsStrings.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\buildproperties.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.InternalsVisibleTo.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.AssemblyInfo.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\sformat.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\sformat.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\sr.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\sr.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ResizeArray.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ResizeArray.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\HashMultiMap.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\HashMultiMap.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\EditDistance.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\EditDistance.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\TaggedCollections.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\TaggedCollections.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\illib.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\illib.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\FileSystem.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\FileSystem.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ildiag.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ildiag.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\zmap.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\zmap.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\zset.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\zset.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\XmlAdapters.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\XmlAdapters.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\InternalCollections.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\InternalCollections.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\QueueList.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\QueueList.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\lib.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\lib.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ImmutableArray.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ImmutableArray.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\rational.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\rational.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\PathMap.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\PathMap.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\RidHelpers.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\range.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\range.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\Logger.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\Logger.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\LanguageFeatures.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\LanguageFeatures.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticOptions.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticOptions.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\TextLayoutRender.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\TextLayoutRender.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticsLogger.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticsLogger.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticResolutionHints.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticResolutionHints.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\prim-lexing.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\prim-lexing.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\prim-parsing.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\prim-parsing.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\ReferenceResolver.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\ReferenceResolver.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\SimulatedMSBuildReferenceResolver.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\SimulatedMSBuildReferenceResolver.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\CompilerLocation.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\CompilerLocation.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\BuildGraph.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\BuildGraph.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\il.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\il.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilx.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilx.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilascii.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilascii.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\ilpars.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\illex.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilprint.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilprint.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilmorph.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilmorph.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilsign.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilsign.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilnativeres.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilnativeres.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilsupp.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilsupp.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilbinary.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilbinary.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilread.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilread.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilwritepdb.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilwritepdb.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilwrite.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilwrite.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilreflect.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilreflect.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\PrettyNaming.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\PrettyNaming.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\UnicodeLexing.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\UnicodeLexing.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\XmlDoc.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\XmlDoc.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTrivia.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTrivia.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTree.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTree.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTreeOps.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTreeOps.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\ParseHelpers.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\ParseHelpers.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pppars.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pars.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\LexHelpers.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\LexHelpers.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pplex.fs" + @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\\lex.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\LexFilter.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\LexFilter.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\tainted.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\tainted.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypeProviders.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypeProviders.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\QuotationPickler.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\QuotationPickler.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\CompilerGlobalState.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\CompilerGlobalState.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTree.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTree.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreeBasics.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreeBasics.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TcGlobals.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreeOps.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreeOps.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreePickle.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreePickle.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\import.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\import.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\TypeHierarchy.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\TypeHierarchy.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\infos.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\infos.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AccessibilityLogic.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AccessibilityLogic.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AttributeChecking.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AttributeChecking.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\TypeRelations.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\TypeRelations.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\InfoReader.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\InfoReader.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\NicePrint.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\NicePrint.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AugmentWithHashCompare.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AugmentWithHashCompare.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\NameResolution.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\NameResolution.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\SignatureConformance.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\SignatureConformance.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\MethodOverrides.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\MethodOverrides.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\MethodCalls.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\MethodCalls.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\PatternMatchCompilation.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\PatternMatchCompilation.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\ConstraintSolver.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\ConstraintSolver.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckFormatStrings.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckFormatStrings.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\FindUnsolved.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\FindUnsolved.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\QuotationTranslator.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\QuotationTranslator.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\PostInferenceChecks.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\PostInferenceChecks.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckBasics.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckBasics.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckExpressions.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckExpressions.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckPatterns.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckPatterns.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckComputationExpressions.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckComputationExpressions.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckIncrementalClasses.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckIncrementalClasses.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckDeclarations.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckDeclarations.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\Optimizer.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\Optimizer.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\DetupleArgs.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\DetupleArgs.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\InnerLambdasToTopLevelFuncs.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\InnerLambdasToTopLevelFuncs.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerCalls.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerCalls.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerSequences.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerSequences.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerComputedCollections.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerComputedCollections.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerStateMachines.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerStateMachines.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerLocalMutables.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerLocalMutables.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\EraseClosures.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\EraseClosures.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\EraseUnions.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\EraseUnions.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\IlxGen.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\IlxGen.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\FxResolver.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\FxResolver.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/AssemblyResolveHandler.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/AssemblyResolveHandler.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/NativeDllResolveHandler.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/NativeDllResolveHandler.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/DependencyProvider.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/DependencyProvider.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerConfig.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerConfig.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerImports.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerImports.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerDiagnostics.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerDiagnostics.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\ParseAndCheckInputs.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\ParseAndCheckInputs.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\ScriptClosure.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\ScriptClosure.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerOptions.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerOptions.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\OptimizeInputs.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\OptimizeInputs.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\XmlDocFileWriter.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\XmlDocFileWriter.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\BinaryResourceFormats.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\BinaryResourceFormats.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\StaticLinking.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\StaticLinking.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CreateILModule.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CreateILModule.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\fsc.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\fsc.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\FSharpDiagnostic.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\FSharpDiagnostic.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\SymbolHelpers.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\SymbolHelpers.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\Symbols.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\Symbols.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\Exprs.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\Exprs.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\SymbolPatterns.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\SymbolPatterns.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\SemanticClassification.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\SemanticClassification.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ItemKey.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ItemKey.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\SemanticClassificationKey.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\SemanticClassificationKey.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpSource.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpSource.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\IncrementalBuild.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\IncrementalBuild.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceCompilerDiagnostics.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceCompilerDiagnostics.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceConstants.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceDeclarationLists.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceDeclarationLists.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceLexing.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceLexing.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParseTreeWalk.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParseTreeWalk.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceNavigation.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceNavigation.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParamInfoLocations.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParamInfoLocations.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpParseFileResults.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpParseFileResults.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParsedInputOps.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParsedInputOps.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceAssemblyContent.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceAssemblyContent.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceXmlDocParser.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceXmlDocParser.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ExternalSymbol.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ExternalSymbol.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\QuickParse.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\QuickParse.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpCheckerResults.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpCheckerResults.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\service.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\service.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceInterfaceStubGenerator.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceInterfaceStubGenerator.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceStructure.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceStructure.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceAnalysis.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceAnalysis.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Interactive\ControlledExecution.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Interactive\fsi.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Interactive\fsi.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Legacy\LegacyMSBuildReferenceResolver.fsi" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Legacy\LegacyMSBuildReferenceResolver.fs" + @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Legacy\LegacyHostedCompilerForTesting.fs" + |] + +[] +let ``FCS for realzies`` () = mkGraphAndReport fcsFiles + +[] +let ``FCS for debugging`` () = + let filesWithAST = + fcsFiles + |> Array.mapi (fun idx file -> + { + Idx = idx + AST = parseSourceCode (file, System.IO.File.ReadAllText(file)) + File = file + }) + + let contents = + Array.map + (fun (file: FileWithAST) -> + printfn "Start %s" file.File + FileContentMapping.mkFileContent file) + filesWithAST + + ignore contents diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs index 064f476697..bd62c26d85 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs @@ -59,7 +59,10 @@ let visitSynModuleDecl (decl: SynModuleDecl) : FileContentEntry list = | SynModuleDecl.Let (bindings = bindings) -> lc visitBinding bindings | SynModuleDecl.Types (typeDefns = typeDefns) -> lc visitSynTypeDefn typeDefns | SynModuleDecl.HashDirective _ -> [] - | SynModuleDecl.ModuleAbbrev _ -> failwith "no support for module abbreviations" + | SynModuleDecl.ModuleAbbrev (longId = longId) -> + // I believe this is enough + // A module abbreviation doesn't appear to be exposed as part of the current module/namespace + visitLongIdent longId | SynModuleDecl.NamespaceFragment _ -> [] | SynModuleDecl.Exception(exnDefn = SynExceptionDefn (exnRepr = SynExceptionDefnRepr (attributes = attributes caseName = caseName diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs index 4c772a90fb..7728063f19 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs @@ -1,17 +1,9 @@ module ParallelTypeCheckingTests.Code.TrieApproach.TrieMapping open System.Collections.Generic -open System.Diagnostics open FSharp.Compiler.Syntax open Microsoft.FSharp.Collections -let time msg f a = - let sw = Stopwatch.StartNew() - let result = f a - sw.Stop() - printfn $"{msg} took %A{sw.Elapsed}" - result - let mergeTrieNodes (defaultChildSize: int) (tries: TrieNode seq) = let rec mergeTrieNodesAux (root: TrieNode) (KeyValue (k, v)) = if root.Children.ContainsKey k then @@ -152,7 +144,7 @@ let rec mkTrieNodeFor (file: FileWithAST) : TrieNode = else TrieNodeInfo.Module(name, file.File) - let children = List.choose (mkTrieForNestedModule file.File) decls + let children = List.choose (mkTrieForSynModuleDecl file.File) decls continuation ( Dictionary<_, _>( @@ -183,11 +175,11 @@ let rec mkTrieNodeFor (file: FileWithAST) : TrieNode = Some { Current = Root; Children = rootNode }) |> mergeTrieNodes contents.Length -and mkTrieForNestedModule file (decl: SynModuleDecl) : KeyValuePair option = +and mkTrieForSynModuleDecl file (decl: SynModuleDecl) : KeyValuePair option = match decl with | SynModuleDecl.NestedModule (moduleInfo = SynComponentInfo(longId = [ nestedModuleIdent ]); decls = decls) -> let name = nestedModuleIdent.idText - let children = List.choose (mkTrieForNestedModule file) decls + let children = List.choose (mkTrieForSynModuleDecl file) decls Some( KeyValuePair( @@ -199,6 +191,8 @@ and mkTrieForNestedModule file (decl: SynModuleDecl) : KeyValuePair + | _ -> None and mkTrieForNestedSigModule file (decl: SynModuleSigDecl) : KeyValuePair option = @@ -220,8 +214,7 @@ and mkTrieForNestedSigModule file (decl: SynModuleSigDecl) : KeyValuePair None let mkTrie (files: FileWithAST array) : TrieNode = - let tries = Array.Parallel.map mkTrieNodeFor files - time "mergeTrieNodes" (mergeTrieNodes 0) tries + mergeTrieNodes 0 (Array.Parallel.map mkTrieNodeFor files) // ================================================================================================================================================== // ================================================================================================================================================== diff --git a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj index fba8557db3..bc17f10a8b 100644 --- a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj +++ b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj @@ -2,7 +2,7 @@ - net7.0 + net7.0 Library false true @@ -10,7 +10,7 @@ true true false - false + true $(OtherFlags) --warnon:1182 $(NoWarn);FS0988;FS1182 $(DefineConstants);RELEASE From 368580180457febfd18875a8a5be69d82bf0d25a Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 17 Nov 2022 16:26:19 +0100 Subject: [PATCH 05/21] queryTrieMemoized --- .../Code/TrieApproach/DependencyResolution.fs | 35 +++++++++++-------- .../Code/TrieApproach/SampleData.fs | 6 ++-- .../Code/TrieApproach/Types.fs | 2 ++ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs index bc0d5b8224..710af4fcb2 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs @@ -34,13 +34,16 @@ let queryTrie (trie: TrieNode) (path: ModuleSegment list) : QueryTrieNodeResult visit trie path +let queryTrieMemoized (trie: TrieNode) : QueryTrie = + Internal.Utilities.Library.Tables.memoize (queryTrie trie) + // Now how to detect the deps between files? // Process the content of each file using some state // Helper function to process a open statement // The statement could link to files and/or should be tracked as an open namespace -let processOpenPath (trie: TrieNode) (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = - let queryResult = queryTrie trie path +let processOpenPath (queryTrie: QueryTrie) (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = + let queryResult = queryTrie path match queryResult with | QueryTrieNodeResult.NodeDoesNotExist -> state @@ -48,8 +51,8 @@ let processOpenPath (trie: TrieNode) (path: ModuleSegment list) (state: FileCont | QueryTrieNodeResult.NodeExposesData files -> state.AddDependenciesAndOpenNamespace(files, path) // Helper function to process an identifier -let processIdentifier (trie: TrieNode) (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = - let queryResult = queryTrie trie path +let processIdentifier (queryTrie: QueryTrie) (path: ModuleSegment list) (state: FileContentQueryState) : FileContentQueryState = + let queryResult = queryTrie path match queryResult with | QueryTrieNodeResult.NodeDoesNotExist -> state @@ -60,38 +63,38 @@ let processIdentifier (trie: TrieNode) (path: ModuleSegment list) (state: FileCo | QueryTrieNodeResult.NodeExposesData files -> state.AddDependencies files // Typically used to folder FileContentEntry items over a FileContentQueryState -let rec processStateEntry (trie: TrieNode) (state: FileContentQueryState) (entry: FileContentEntry) : FileContentQueryState = +let rec processStateEntry (queryTrie: QueryTrie) (state: FileContentQueryState) (entry: FileContentEntry) : FileContentQueryState = match entry with | FileContentEntry.TopLevelNamespace (topLevelPath, content) -> let state = match topLevelPath with | [] -> state - | _ -> processOpenPath trie topLevelPath state + | _ -> processOpenPath queryTrie topLevelPath state - List.fold (processStateEntry trie) state content + List.fold (processStateEntry queryTrie) state content | FileContentEntry.OpenStatement path -> // An open statement can directly reference file or be a partial open statement // Both cases need to be processed. - let stateAfterFullOpenPath = processOpenPath trie path state + let stateAfterFullOpenPath = processOpenPath queryTrie path state // Any existing open statement could be extended with the current path (if that node where to exists in the trie) // The extended path could add a new link (in case of a module or namespace with types) // It might also not add anything at all (in case it the extended path is still a partial one) (stateAfterFullOpenPath, state.OpenNamespaces) - ||> Seq.fold (fun acc openNS -> processOpenPath trie [ yield! openNS; yield! path ] acc) + ||> Seq.fold (fun acc openNS -> processOpenPath queryTrie [ yield! openNS; yield! path ] acc) | FileContentEntry.PrefixedIdentifier path -> // process the name was if it were a FQN - let stateAfterFullIdentifier = processIdentifier trie path state + let stateAfterFullIdentifier = processIdentifier queryTrie path state // Process the name in combination with the existing open namespaces (stateAfterFullIdentifier, state.OpenNamespaces) - ||> Seq.fold (fun acc openNS -> processIdentifier trie [ yield! openNS; yield! path ] acc) + ||> Seq.fold (fun acc openNS -> processIdentifier queryTrie [ yield! openNS; yield! path ] acc) | FileContentEntry.NestedModule (nestedContent = nestedContent) -> // We don't want our current state to be affect by any open statements in the nested module - let nestedState = List.fold (processStateEntry trie) state nestedContent + let nestedState = List.fold (processStateEntry queryTrie) state nestedContent // Afterward we are only interested in the found dependencies in the nested module let foundDependencies = Set.union state.FoundDependencies nestedState.FoundDependencies @@ -121,7 +124,10 @@ let mkGraph (files: FileWithAST array) = time "TrieMapping.mkTrie" TrieMapping.mkTrie input - let fileContents = Array.Parallel.map FileContentMapping.mkFileContent files + let queryTrie: QueryTrie = queryTrieMemoized trie + + let fileContents = + time "FileContentMapping.mkFileContent" Array.Parallel.map FileContentMapping.mkFileContent files time "mkGraph" @@ -131,7 +137,7 @@ let mkGraph (files: FileWithAST array) = let knownFiles = getFileNameBefore files file.Idx let result = - Seq.fold (processStateEntry trie) (FileContentQueryState.Create file.File knownFiles) fileContent + Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create file.File knownFiles) fileContent file.File, Set.toArray result.FoundDependencies) files @@ -521,7 +527,6 @@ let ``FCS for debugging`` () = let contents = Array.map (fun (file: FileWithAST) -> - printfn "Start %s" file.File FileContentMapping.mkFileContent file) filesWithAST diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs index e598e337b7..bcf7cf388c 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs @@ -729,8 +729,10 @@ let ``Full project simulation`` () = let knownFiles = files.[0 .. (fileContent.Idx - 1)] |> Array.map (fun f -> f.Name) |> set + let queryTrie: QueryTrie = queryTrieMemoized fantomasCoreTrie + let result = - Seq.fold (processStateEntry fantomasCoreTrie) (FileContentQueryState.Create fileContent.Name knownFiles) fileContent.Content + Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create fileContent.Name knownFiles) fileContent.Content fileContent.Name, Set.toArray result.FoundDependencies) @@ -785,7 +787,7 @@ let ``ProcessOpenStatement full path match`` () = |]) let result = - processOpenPath fantomasCoreTrie [ "Fantomas"; "Core"; "AstExtensions" ] state + processOpenPath (queryTrie fantomasCoreTrie) [ "Fantomas"; "Core"; "AstExtensions" ] state let dep = Seq.exactlyOne result.FoundDependencies Assert.AreEqual("AstExtensions.fsi", dep) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs index f7bccabcff..d03929059a 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs @@ -105,3 +105,5 @@ type QueryTrieNodeResult = | NodeDoesNotExposeData /// A node was found with one or more file links | NodeExposesData of Files + +type QueryTrie = ModuleSegment list -> QueryTrieNodeResult From 94a77573da41e13af253d8f933ab99bdbbe6d48d Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 17 Nov 2022 18:12:29 +0100 Subject: [PATCH 06/21] Call continuation for SynExpr.Record. --- .../Code/TrieApproach/FileContentMapping.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs index bd62c26d85..c2f98f9c79 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/FileContentMapping.fs @@ -351,7 +351,7 @@ let visitSynExpr (e: SynExpr) : FileContentEntry list = | Some (t, e, _, _, _), None -> visit e (fun nodes -> [ yield! visitSynType t; yield! nodes; yield! fieldNodes ] |> continuation) | None, Some (e, _) -> visit e (fun nodes -> nodes @ fieldNodes |> continuation) - | _ -> fieldNodes + | _ -> continuation fieldNodes | SynExpr.New (targetType = targetType; expr = expr) -> visit expr (fun nodes -> visitSynType targetType @ nodes |> continuation) | SynExpr.ObjExpr (objType, argOptions, _, bindings, members, extraImpls, _, _) -> [ @@ -684,7 +684,7 @@ module Tests = [] let ``Test a single file`` () = let fileName = - @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fsi" + @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fs" let ast = parseSourceCode (fileName, System.IO.File.ReadAllText(fileName)) let contents = mkFileContent { Idx = 0; File = fileName; AST = ast } From 966698f817033117b3b4eaca071d42733535242d Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 17 Nov 2022 18:13:37 +0100 Subject: [PATCH 07/21] Test all lengths of PrefixedIdentifier path. --- .../Code/TrieApproach/DependencyResolution.fs | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs index 710af4fcb2..a4a59db495 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs @@ -85,12 +85,21 @@ let rec processStateEntry (queryTrie: QueryTrie) (state: FileContentQueryState) ||> Seq.fold (fun acc openNS -> processOpenPath queryTrie [ yield! openNS; yield! path ] acc) | FileContentEntry.PrefixedIdentifier path -> - // process the name was if it were a FQN - let stateAfterFullIdentifier = processIdentifier queryTrie path state - - // Process the name in combination with the existing open namespaces - (stateAfterFullIdentifier, state.OpenNamespaces) - ||> Seq.fold (fun acc openNS -> processIdentifier queryTrie [ yield! openNS; yield! path ] acc) + match path with + | [] -> + // should not be possible though + state + | _ -> + // path could consist out of multiple segments + (state, [| 1 .. path.Length |]) + ||> Seq.fold (fun state takeParts -> + let path = List.take takeParts path + // process the name was if it were a FQN + let stateAfterFullIdentifier = processIdentifier queryTrie path state + + // Process the name in combination with the existing open namespaces + (stateAfterFullIdentifier, state.OpenNamespaces) + ||> Seq.fold (fun acc openNS -> processIdentifier queryTrie [ yield! openNS; yield! path ] acc)) | FileContentEntry.NestedModule (nestedContent = nestedContent) -> // We don't want our current state to be affect by any open statements in the nested module @@ -139,7 +148,7 @@ let mkGraph (files: FileWithAST array) = let result = Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create file.File knownFiles) fileContent - file.File, Set.toArray result.FoundDependencies) + file, Set.toArray result.FoundDependencies) files // ============================================================================================================= @@ -164,9 +173,9 @@ let mkGraphAndReport files = let depString = String.concat "\n " deps if deps.Length = 0 then - printfn $"%s{fileName}: []" + printfn $"%s{fileName.File}: []" else - printfn $"%s{fileName}:\n {depString}" + printfn $"%s{fileName.File}:\n {depString}" [] let ``Fantomas.Core for realzies`` () = @@ -525,9 +534,6 @@ let ``FCS for debugging`` () = }) let contents = - Array.map - (fun (file: FileWithAST) -> - FileContentMapping.mkFileContent file) - filesWithAST + Array.map (fun (file: FileWithAST) -> FileContentMapping.mkFileContent file) filesWithAST ignore contents From 9774616e090d6bcdb8029fd023b47ca09142111a Mon Sep 17 00:00:00 2001 From: nojaf Date: Fri, 18 Nov 2022 12:45:53 +0100 Subject: [PATCH 08/21] Use indexes in TrieNodeInfo, consider auto open modules and do some stricter checks in the TypedTreeGraph. --- .../Code/TrieApproach/AutoOpenDetection.fs | 71 +++++ .../Code/TrieApproach/DependencyResolution.fs | 41 ++- .../Code/TrieApproach/SampleData.fs | 262 +++++++++--------- .../Code/TrieApproach/TrieMapping.fs | 27 +- .../Code/TrieApproach/Types.fs | 22 +- .../ParallelTypeCheckingTests.fsproj | 1 + .../Tests/TypedTreeGraph.fs | 131 +++++++-- 7 files changed, 366 insertions(+), 189 deletions(-) create mode 100644 tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs new file mode 100644 index 0000000000..d97bcf93c5 --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs @@ -0,0 +1,71 @@ +module ParallelTypeCheckingTests.Code.TrieApproach.AutoOpenDetection + +open FSharp.Compiler.Syntax + +let private autoOpenShapes = + set + [| + "FSharp.Core.AutoOpenAttribute" + "Core.AutoOpenAttribute" + "AutoOpenAttribute" + "FSharp.Core.AutoOpen" + "Core.AutoOpen" + "AutoOpen" + |] + +/// This isn't bullet proof but I wonder who would really alias this very core attribute. +let isAutoOpenAttribute (attribute: SynAttribute) = + match attribute.ArgExpr with + | SynExpr.Const(constant = SynConst.Unit _) + | SynExpr.Const(constant = SynConst.String _) + | SynExpr.Paren(expr = SynExpr.Const(constant = SynConst.String _)) -> + let attributeName = + attribute.TypeName.LongIdent + |> List.map (fun ident -> ident.idText) + |> String.concat "." + + autoOpenShapes.Contains attributeName + | _ -> false + +let isAnyAttributeAutoOpen (attributes: SynAttributes) = + List.exists (fun (atl: SynAttributeList) -> List.exists isAutoOpenAttribute atl.Attributes) attributes + +let rec hasNestedModuleWithAutoOpenAttribute (decls: SynModuleDecl list) : bool = + decls + |> List.exists (function + | SynModuleDecl.NestedModule (moduleInfo = SynComponentInfo (attributes = attributes); decls = decls) -> + isAnyAttributeAutoOpen attributes || hasNestedModuleWithAutoOpenAttribute decls + | _ -> false) + +let rec hasNestedSigModuleWithAutoOpenAttribute (decls: SynModuleSigDecl list) : bool = + decls + |> List.exists (function + | SynModuleSigDecl.NestedModule (moduleInfo = SynComponentInfo (attributes = attributes); moduleDecls = decls) -> + isAnyAttributeAutoOpen attributes + || hasNestedSigModuleWithAutoOpenAttribute decls + | _ -> false) + +let hasAutoOpenAttributeInFile (ast: ParsedInput) : bool = + match ast with + | ParsedInput.SigFile (ParsedSigFileInput (contents = contents)) -> + contents + |> List.exists (fun (SynModuleOrNamespaceSig (attribs = attribs; decls = decls)) -> + isAnyAttributeAutoOpen attribs || hasNestedSigModuleWithAutoOpenAttribute decls) + | ParsedInput.ImplFile (ParsedImplFileInput (contents = contents)) -> + contents + |> List.exists (fun (SynModuleOrNamespace (attribs = attribs; decls = decls)) -> + isAnyAttributeAutoOpen attribs || hasNestedModuleWithAutoOpenAttribute decls) + +// ============================================================================================================================== +// ============================================================================================================================== + +open NUnit.Framework +open FSharp.Compiler.Service.Tests.Common + +[] +let ``detect auto open`` () = + let file = + @"C:\Users\nojaf\Projects\safesparrow-fsharp\src\Compiler\Utilities\ImmutableArray.fsi" + + let ast = parseSourceCode (file, System.IO.File.ReadAllText(file)) + Assert.True(hasAutoOpenAttributeInFile ast) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs index a4a59db495..39177fdcdc 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs @@ -1,5 +1,6 @@ module ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution +open System.Linq open FSharp.Compiler.Syntax // This is pseudo code of how we could restructure the trie code @@ -113,7 +114,7 @@ let rec processStateEntry (queryTrie: QueryTrie) (state: FileContentQueryState) } let getFileNameBefore (files: FileWithAST array) idx = - files.[0 .. (idx - 1)] |> Array.map (fun f -> f.File) |> Set.ofArray + files.[0 .. (idx - 1)] |> Array.map (fun f -> f.Idx) |> Set.ofArray let time msg f a = let sw = System.Diagnostics.Stopwatch.StartNew() @@ -123,21 +124,25 @@ let time msg f a = result let mkGraph (files: FileWithAST array) = - let trie = - let input = - files - |> Array.filter (fun f -> - match f.AST with - | ParsedInput.SigFile _ -> true - | ParsedInput.ImplFile _ -> Array.forall (fun (sigFile: FileWithAST) -> sigFile.File <> $"{f.File}i") files) + let trieInput = + files + |> Array.filter (fun f -> + match f.AST with + | ParsedInput.SigFile _ -> true + | ParsedInput.ImplFile _ -> Array.forall (fun (sigFile: FileWithAST) -> sigFile.File <> $"{f.File}i") files) - time "TrieMapping.mkTrie" TrieMapping.mkTrie input + let trie = time "TrieMapping.mkTrie" TrieMapping.mkTrie trieInput let queryTrie: QueryTrie = queryTrieMemoized trie let fileContents = time "FileContentMapping.mkFileContent" Array.Parallel.map FileContentMapping.mkFileContent files + let filesWithAutoOpen = + trieInput + |> Array.filter (fun f -> AutoOpenDetection.hasAutoOpenAttributeInFile f.AST) + |> Array.map (fun f -> f.Idx) + time "mkGraph" Array.Parallel.map @@ -146,9 +151,18 @@ let mkGraph (files: FileWithAST array) = let knownFiles = getFileNameBefore files file.Idx let result = - Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create file.File knownFiles) fileContent + Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create file.Idx knownFiles) fileContent + + let allDependencies = + if filesWithAutoOpen.Length > 0 then + let autoOpenDependencies = + set ([| 0 .. (file.Idx - 1) |].Intersect(filesWithAutoOpen)) + + Set.union result.FoundDependencies autoOpenDependencies + else + result.FoundDependencies - file, Set.toArray result.FoundDependencies) + file, Set.toArray allDependencies) files // ============================================================================================================= @@ -170,7 +184,10 @@ let mkGraphAndReport files = let graph = mkGraph filesWithAST for fileName, deps in graph do - let depString = String.concat "\n " deps + let depString = + deps + |> Array.map (fun depIdx -> filesWithAST.[depIdx].File) + |> String.concat "\n " if deps.Length = 0 then printfn $"%s{fileName.File}: []" diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs index bcf7cf388c..06d71e63c3 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs @@ -4,121 +4,6 @@ open System.Collections.Generic open NUnit.Framework open ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution -let dictionary<'key, 'value when 'key: equality> (entries: ('key * 'value) seq) = - entries |> Seq.map (fun (k, v) -> KeyValuePair(k, v)) |> Dictionary - -let noChildren = Dictionary(0) -let emptyHS () = HashSet(0) - -// This should be constructed from the AST -let fantomasCoreTrie: TrieNode = - { - Current = TrieNodeInfo.Root - Children = - dictionary - [| - "System", - { - Current = TrieNodeInfo.Namespace("System", emptyHS ()) - Children = - dictionary - [| - "AssemblyVersionInformation", - { - Current = TrieNodeInfo.Module("AssemblyVersionInformation", "AssemblyInfo.fs") - Children = noChildren - } - |] - } - "Fantomas", - { - Current = TrieNodeInfo.Namespace("Fantomas", emptyHS ()) - Children = - dictionary - [| - "Core", - { - Current = TrieNodeInfo.Namespace("Core", emptyHS ()) - Children = - dictionary - [| - "ISourceTextExtensions", - { - Current = TrieNodeInfo.Module("ISourceTextExtensions", "ISourceTextExtensions.fs") - Children = noChildren - } - "RangeHelpers", - { - Current = TrieNodeInfo.Module("RangeHelpers", "RangeHelpers.fs") - Children = noChildren - } - "RangePatterns", - { - Current = TrieNodeInfo.Module("RangePatterns", "RangeHelpers.fs") - Children = noChildren - } - "AstExtensions", - { - Current = TrieNodeInfo.Module("AstExtensions", "AstExtensions.fsi") - Children = noChildren - } - "TriviaTypes", - { - Current = TrieNodeInfo.Module("TriviaTypes", "TriviaTypes.fs") - Children = noChildren - } - "Char", - { - Current = TrieNodeInfo.Module("Char", "Utils.fs") - Children = noChildren - } - "String", - { - Current = TrieNodeInfo.Module("String", "Utils.fs") - Children = noChildren - } - "Cache", - { - Current = TrieNodeInfo.Module("Cache", "Utils.fs") - Children = noChildren - } - "Dict", - { - Current = TrieNodeInfo.Module("Dict", "Utils.fs") - Children = noChildren - } - "List", - { - Current = TrieNodeInfo.Module("List", "Utils.fs") - Children = noChildren - } - "Map", - { - Current = TrieNodeInfo.Module("Map", "Utils.fs") - Children = noChildren - } - "Async", - { - Current = TrieNodeInfo.Module("Async", "Utils.fs") - Children = noChildren - } - "Continuation", - { - Current = TrieNodeInfo.Module("Continuation", "Utils.fs") - Children = noChildren - } - "SourceParser", - { - Current = TrieNodeInfo.Module("SourceParser", "SourceParser.fs") - Children = noChildren - } - |] - } - |] - } - |] - } - // Some helper DSL functions to construct the FileContentEntry items // This should again be mapped from the AST @@ -721,23 +606,144 @@ let files = } |] +let dictionary<'key, 'value when 'key: equality> (entries: ('key * 'value) seq) = + entries |> Seq.map (fun (k, v) -> KeyValuePair(k, v)) |> Dictionary + +let noChildren = Dictionary(0) +let emptyHS () = HashSet(0) + +let indexOf name = + Array.find (fun (fc: FileContent) -> fc.Name = name) files |> fun fc -> fc.Idx + +// This should be constructed from the AST +let fantomasCoreTrie: TrieNode = + { + Current = TrieNodeInfo.Root + Children = + dictionary + [| + "System", + { + Current = TrieNodeInfo.Namespace("System", emptyHS ()) + Children = + dictionary + [| + "AssemblyVersionInformation", + { + Current = TrieNodeInfo.Module("AssemblyVersionInformation", indexOf "AssemblyInfo.fs") + Children = noChildren + } + |] + } + "Fantomas", + { + Current = TrieNodeInfo.Namespace("Fantomas", emptyHS ()) + Children = + dictionary + [| + "Core", + { + Current = TrieNodeInfo.Namespace("Core", emptyHS ()) + Children = + dictionary + [| + "ISourceTextExtensions", + { + Current = + TrieNodeInfo.Module("ISourceTextExtensions", indexOf "ISourceTextExtensions.fs") + Children = noChildren + } + "RangeHelpers", + { + Current = TrieNodeInfo.Module("RangeHelpers", indexOf "RangeHelpers.fs") + Children = noChildren + } + "RangePatterns", + { + Current = TrieNodeInfo.Module("RangePatterns", indexOf "RangeHelpers.fs") + Children = noChildren + } + "AstExtensions", + { + Current = TrieNodeInfo.Module("AstExtensions", indexOf "AstExtensions.fsi") + Children = noChildren + } + "TriviaTypes", + { + Current = TrieNodeInfo.Module("TriviaTypes", indexOf "TriviaTypes.fs") + Children = noChildren + } + "Char", + { + Current = TrieNodeInfo.Module("Char", indexOf "Utils.fs") + Children = noChildren + } + "String", + { + Current = TrieNodeInfo.Module("String", indexOf "Utils.fs") + Children = noChildren + } + "Cache", + { + Current = TrieNodeInfo.Module("Cache", indexOf "Utils.fs") + Children = noChildren + } + "Dict", + { + Current = TrieNodeInfo.Module("Dict", indexOf "Utils.fs") + Children = noChildren + } + "List", + { + Current = TrieNodeInfo.Module("List", indexOf "Utils.fs") + Children = noChildren + } + "Map", + { + Current = TrieNodeInfo.Module("Map", indexOf "Utils.fs") + Children = noChildren + } + "Async", + { + Current = TrieNodeInfo.Module("Async", indexOf "Utils.fs") + Children = noChildren + } + "Continuation", + { + Current = TrieNodeInfo.Module("Continuation", indexOf "Utils.fs") + Children = noChildren + } + "SourceParser", + { + Current = TrieNodeInfo.Module("SourceParser", indexOf "SourceParser.fs") + Children = noChildren + } + |] + } + |] + } + |] + } + [] let ``Full project simulation`` () = let graph = files |> Array.map (fun fileContent -> let knownFiles = - files.[0 .. (fileContent.Idx - 1)] |> Array.map (fun f -> f.Name) |> set + files.[0 .. (fileContent.Idx - 1)] |> Array.map (fun f -> f.Idx) |> set let queryTrie: QueryTrie = queryTrieMemoized fantomasCoreTrie - + let result = - Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create fileContent.Name knownFiles) fileContent.Content + Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create fileContent.Idx knownFiles) fileContent.Content fileContent.Name, Set.toArray result.FoundDependencies) for fileName, deps in graph do - let depString = String.concat ", " deps + let depString = + deps |> Array.map (fun depIdx -> files.[depIdx].Name) |> String.concat ", " + printfn $"%s{fileName}: [{depString}]" [] @@ -765,7 +771,7 @@ let ``Query module node that exposes one file`` () = match result with | QueryTrieNodeResult.NodeExposesData file -> let file = Seq.exactlyOne file - Assert.AreEqual("ISourceTextExtensions.fs", file) + Assert.AreEqual(indexOf "ISourceTextExtensions.fs", file) | result -> Assert.Fail $"Unexpected result: %A{result}" [] @@ -775,22 +781,22 @@ let ``ProcessOpenStatement full path match`` () = let state = FileContentQueryState.Create - sourceParser.Name + sourceParser.Idx (set [| - "AssemblyInfo.fs" - "ISourceTextExtensions.fs" - "RangeHelpers.fs" - "AstExtensions.fsi" - "TriviaTypes.fs" - "Utils.fs" + indexOf "AssemblyInfo.fs" + indexOf "ISourceTextExtensions.fs" + indexOf "RangeHelpers.fs" + indexOf "AstExtensions.fsi" + indexOf "TriviaTypes.fs" + indexOf "Utils.fs" |]) let result = processOpenPath (queryTrie fantomasCoreTrie) [ "Fantomas"; "Core"; "AstExtensions" ] state let dep = Seq.exactlyOne result.FoundDependencies - Assert.AreEqual("AstExtensions.fsi", dep) + Assert.AreEqual(indexOf "AstExtensions.fsi", dep) #if INTERACTIVE open System.Text.RegularExpressions diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs index 7728063f19..efcbebb0fe 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs @@ -77,11 +77,11 @@ let rec mkTrieNodeFor (file: FileWithAST) : TrieNode = let current = if isNamespace then - TrieNodeInfo.Namespace(name, (if hasTypes then hs file.File else emptyHS ())) + TrieNodeInfo.Namespace(name, (if hasTypes then hs file.Idx else emptyHS ())) else - TrieNodeInfo.Module(name, file.File) + TrieNodeInfo.Module(name, file.Idx) - let children = List.choose (mkTrieForNestedSigModule file.File) decls + let children = List.choose (mkTrieForNestedSigModule file.Idx) decls continuation ( Dictionary<_, _>( @@ -140,11 +140,11 @@ let rec mkTrieNodeFor (file: FileWithAST) : TrieNode = let current = if isNamespace then - TrieNodeInfo.Namespace(name, (if hasTypes then hs file.File else emptyHS ())) + TrieNodeInfo.Namespace(name, (if hasTypes then hs file.Idx else emptyHS ())) else - TrieNodeInfo.Module(name, file.File) + TrieNodeInfo.Module(name, file.Idx) - let children = List.choose (mkTrieForSynModuleDecl file.File) decls + let children = List.choose (mkTrieForSynModuleDecl file.Idx) decls continuation ( Dictionary<_, _>( @@ -175,37 +175,34 @@ let rec mkTrieNodeFor (file: FileWithAST) : TrieNode = Some { Current = Root; Children = rootNode }) |> mergeTrieNodes contents.Length -and mkTrieForSynModuleDecl file (decl: SynModuleDecl) : KeyValuePair option = +and mkTrieForSynModuleDecl (fileIndex: int) (decl: SynModuleDecl) : KeyValuePair option = match decl with | SynModuleDecl.NestedModule (moduleInfo = SynComponentInfo(longId = [ nestedModuleIdent ]); decls = decls) -> let name = nestedModuleIdent.idText - let children = List.choose (mkTrieForSynModuleDecl file) decls + let children = List.choose (mkTrieForSynModuleDecl fileIndex) decls Some( KeyValuePair( name, { - Current = TrieNodeInfo.Module(name, file) + Current = TrieNodeInfo.Module(name, fileIndex) Children = Dictionary(children) } ) ) - - // | SynModuleDecl.ModuleAbbrev(ident = ident) -> - | _ -> None -and mkTrieForNestedSigModule file (decl: SynModuleSigDecl) : KeyValuePair option = +and mkTrieForNestedSigModule (fileIndex: int) (decl: SynModuleSigDecl) : KeyValuePair option = match decl with | SynModuleSigDecl.NestedModule (moduleInfo = SynComponentInfo(longId = [ nestedModuleIdent ]); moduleDecls = decls) -> let name = nestedModuleIdent.idText - let children = List.choose (mkTrieForNestedSigModule file) decls + let children = List.choose (mkTrieForNestedSigModule fileIndex) decls Some( KeyValuePair( name, { - Current = TrieNodeInfo.Module(name, file) + Current = TrieNodeInfo.Module(name, fileIndex) Children = Dictionary(children) } ) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs index d03929059a..34f6be530f 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs @@ -20,8 +20,8 @@ type FileWithAST = /// Children of a namespace don't automatically depend on each other for that reason type TrieNodeInfo = | Root - | Module of segment: string * file: File - | Namespace of segment: string * filesThatExposeTypes: HashSet + | Module of segment: string * file: int + | Namespace of segment: string * filesThatExposeTypes: HashSet member x.Segment = match x with @@ -29,7 +29,7 @@ type TrieNodeInfo = | Module (segment = segment) | Namespace (segment = segment) -> segment - member x.Files: Files = + member x.Files: Set = match x with | Root -> failwith "Root has no files" | Module (file = file) -> Set.singleton file @@ -66,20 +66,20 @@ type FileContent = type FileContentQueryState = { OpenNamespaces: Set - FoundDependencies: Set - CurrentFile: File - KnownFiles: Files + FoundDependencies: Set + CurrentFile: int + KnownFiles: Set } - static member Create (file: File) (knownFiles: Files) = + static member Create (fileIndex: int) (knownFiles: Set) = { OpenNamespaces = Set.empty FoundDependencies = Set.empty - CurrentFile = file + CurrentFile = fileIndex KnownFiles = knownFiles } - member x.AddDependencies(files: Files) : FileContentQueryState = + member x.AddDependencies(files: Set) : FileContentQueryState = let files = Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies { x with FoundDependencies = files } @@ -88,7 +88,7 @@ type FileContentQueryState = OpenNamespaces = Set.add path x.OpenNamespaces } - member x.AddDependenciesAndOpenNamespace(files: Files, path: ModuleSegment list) = + member x.AddDependenciesAndOpenNamespace(files: Set, path: ModuleSegment list) = let foundDependencies = Set.filter x.KnownFiles.Contains files |> Set.union x.FoundDependencies @@ -104,6 +104,6 @@ type QueryTrieNodeResult = /// A node was found but it yielded no file links | NodeDoesNotExposeData /// A node was found with one or more file links - | NodeExposesData of Files + | NodeExposesData of Set type QueryTrie = ModuleSegment list -> QueryTrieNodeResult diff --git a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj index bc17f10a8b..19069edd21 100644 --- a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj +++ b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj @@ -40,6 +40,7 @@ + diff --git a/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs b/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs index acac738426..35b2f843c6 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs @@ -21,15 +21,15 @@ let codebases = WorkDir = $@"{__SOURCE_DIRECTORY__}\.fcs_test\src\compiler" Path = $@"{__SOURCE_DIRECTORY__}\FCS.args.txt" } + // { + // WorkDir = $@"{__SOURCE_DIRECTORY__}\.fcs_test\tests\FSharp.Compiler.ComponentTests" + // Path = $@"{__SOURCE_DIRECTORY__}\ComponentTests.args.txt" + // } + // Hard coded example ;) { - WorkDir = $@"{__SOURCE_DIRECTORY__}\.fcs_test\tests\FSharp.Compiler.ComponentTests" - Path = $@"{__SOURCE_DIRECTORY__}\ComponentTests.args.txt" + WorkDir = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core" + Path = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\args.txt" } - // Hard coded example ;) - // { - // WorkDir = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core" - // Path = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\args.txt" - // } |] let checker = FSharpChecker.Create(keepAssemblyContents = true) @@ -38,9 +38,14 @@ type DepCollector(projectRoot: string, projectFile: string) = let deps = HashSet() member this.Add(declarationLocation: range) : unit = - let sourceLocation = declarationLocation.FileName + let sourceLocation = Path.GetFullPath declarationLocation.FileName + let ext = Path.GetExtension sourceLocation - if sourceLocation.StartsWith projectRoot && sourceLocation <> projectFile then + if + (ext = ".fs" || ext = ".fsi") + && sourceLocation.StartsWith projectRoot + && sourceLocation <> projectFile + then deps.Add(sourceLocation.Substring(projectRoot.Length + 1)) |> ignore member this.Deps = Seq.toArray deps @@ -76,7 +81,7 @@ let graphFromTypedTree (checker: FSharpChecker) (projectDir: string) (projectOptions: FSharpProjectOptions) - : Async * IReadOnlyDictionary> = + : Async * DepsGraph> = async { let files = Dictionary() @@ -96,7 +101,8 @@ let graphFromTypedTree | FSharpCheckFileAnswer.Aborted -> return failwith "aborted" | FSharpCheckFileAnswer.Succeeded fileResult -> let allSymbols = fileResult.GetAllUsesOfAllSymbolsInFile() |> Seq.toArray - let collector = DepCollector(projectDir, fileName) + let fullPath = Path.GetFullPath fileName + let collector = DepCollector(projectDir, fullPath) for s in allSymbols do collectFromSymbol collector s.Symbol @@ -131,7 +137,6 @@ let ``Create Graph from typed tree`` (code: Codebase) = let previousDir = Environment.CurrentDirectory async { - try Environment.CurrentDirectory <- code.WorkDir @@ -173,6 +178,9 @@ let ``Create Graph from typed tree`` (code: Codebase) = let path = $"{fileName}.typed-tree.deps.json" graphFromTypedTree |> Graph.map (fun n -> n.Name) |> Graph.serialiseToJson path + let fileIndexToName = + files.Values |> Seq.map (fun file -> file.Idx.Idx, file.Name) |> dict + let sourceFiles = files.Values |> Seq.sortBy (fun file -> file.Idx.Idx) @@ -194,25 +202,102 @@ let ``Create Graph from typed tree`` (code: Codebase) = Assert.True(graphFromTypedTree.Count = graphFromHeuristic.Graph.Count, "Both graphs should have the same amount of entries.") - let depNames (files: File array) = - Array.map (fun (f: File) -> Path.GetFileName(f.Name)) files + let depNames (depIdxs: int seq) = + depIdxs + |> Seq.map (fun idx -> fileIndexToName.[idx] |> Path.GetFileName) |> String.concat ", " - for KeyValue (file, deps) in graphFromHeuristic.Graph do - let depsFromTypedTree = graphFromTypedTree.[file] + let collectAllDeps (graph: DepsGraph) = + let getKeyByIdx idx = + graph.Keys |> Seq.find (fun file -> file.Idx.Idx = idx) + + (Map.empty, [ 0 .. (sourceFiles.Length - 1) ]) + ||> List.fold (fun acc idx -> + let deps = graph.[getKeyByIdx idx] + + let allDeps = + set + [| + yield! (Seq.map (fun dep -> dep.Idx.Idx) deps) + yield! (Seq.collect (fun dep -> Map.find dep.Idx.Idx acc) deps) + |] - if Array.isEmpty depsFromTypedTree && not (Array.isEmpty deps) then - printfn $"{file.Name} has %A{(depNames deps)} while the typed tree had none!" + Map.add idx allDeps acc) + + let typedTreeMap = collectAllDeps graphFromTypedTree + let heuristicMap = collectAllDeps graphFromHeuristic.Graph + + let compareDeps source fileName idx (depsFromHeuristic: Set) = + let depsFromTypedTree = Map.find idx typedTreeMap + + if Set.isEmpty depsFromTypedTree && not (Set.isEmpty depsFromHeuristic) then + printfn $"{source}:{fileName} has %A{(depNames depsFromHeuristic)} while the typed tree had none!" else - let isSuperSet = - depsFromTypedTree |> Seq.forall (fun ttDep -> Seq.contains ttDep deps) + let isSuperSet = Set.isSuperset depsFromHeuristic depsFromTypedTree + let delta = Set.difference depsFromTypedTree depsFromHeuristic Assert.IsTrue( isSuperSet, - $"""{file.Name} did not contain a superset of the typed tree dependencies: -Typed tree dependencies: %A{depNames depsFromTypedTree}. -Heuristic dependencies: %A{depNames deps}.""" + $"""{fileName} did not contain a superset of the typed tree dependencies: +{source} is missing dependencies: %A{depNames delta}.""" ) + + graphFromHeuristic.Graph.Keys + |> Seq.toArray + |> Array.Parallel.iter (fun file -> compareDeps "Current heuristic" file.Name file.Idx.Idx (Map.find file.Idx.Idx heuristicMap)) + + let alternativeGraphFromHeuristic = + let sourceFiles = + files.Values + |> Seq.sortBy (fun file -> file.Idx.Idx) + |> Seq.map (fun file -> + let ast = + match file.AST with + | ASTOrFsix.AST ast -> ast + | ASTOrFsix.Fsix _ -> failwith "unexpected fsix" + + { + Idx = file.Idx.Idx + File = file.Name + AST = ast + }: ParallelTypeCheckingTests.Code.TrieApproach.FileWithAST) + |> Seq.toArray + + ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution.mkGraph sourceFiles + + let alternateMap = + let getDepsByIdx idx = + alternativeGraphFromHeuristic + |> Seq.find (fun (file, _) -> file.Idx = idx) + |> snd + + (Map.empty, [ 0 .. (sourceFiles.Length - 1) ]) + ||> List.fold (fun acc idx -> + let deps = getDepsByIdx idx + + let allDeps = + set [| yield! deps; yield! (Seq.collect (fun dep -> Map.find dep acc) deps) |] + + Map.add idx allDeps acc) + + let rec getAllDepsFromAlt idx : Set = + let currentDeps = + alternativeGraphFromHeuristic + |> Seq.find (fun (file, _) -> file.Idx = idx) + |> snd + + let transitiveDeps = Seq.collect getAllDepsFromAltMemoized currentDeps + + set [| yield! currentDeps; yield! transitiveDeps |] + + and getAllDepsFromAltMemoized = + Internal.Utilities.Library.Tables.memoize getAllDepsFromAlt + + Array.Parallel.iter + (fun (file: ParallelTypeCheckingTests.Code.TrieApproach.FileWithAST, _) -> + compareDeps "Alternative heuristic" file.File file.Idx (Map.find file.Idx alternateMap)) + alternativeGraphFromHeuristic + finally Environment.CurrentDirectory <- previousDir } From 1bf0b66a33c4c1b7791fd10691e6e86e15b90bb5 Mon Sep 17 00:00:00 2001 From: nojaf Date: Fri, 18 Nov 2022 13:19:43 +0100 Subject: [PATCH 09/21] Try out the new approach on Fantomas.Core.Tests. --- .../Code/TrieApproach/AutoOpenDetection.fs | 5 +++-- .../Code/TrieApproach/DependencyResolution.fs | 3 +++ .../Code/TrieApproach/SampleData.fs | 19 ++++--------------- .../Code/TrieApproach/TrieMapping.fs | 7 ------- .../Code/TrieApproach/Types.fs | 1 - .../Tests/TypedTreeGraph.fs | 13 +++++++++---- 6 files changed, 19 insertions(+), 29 deletions(-) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs index d97bcf93c5..fbca7b37ba 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs @@ -59,13 +59,14 @@ let hasAutoOpenAttributeInFile (ast: ParsedInput) : bool = // ============================================================================================================================== // ============================================================================================================================== +open System.IO open NUnit.Framework open FSharp.Compiler.Service.Tests.Common [] let ``detect auto open`` () = let file = - @"C:\Users\nojaf\Projects\safesparrow-fsharp\src\Compiler\Utilities\ImmutableArray.fsi" + Path.Combine(__SOURCE_DIRECTORY__, "..", "..", "..", "..", "src", "Compiler", "Utilities", "ImmutableArray.fsi") - let ast = parseSourceCode (file, System.IO.File.ReadAllText(file)) + let ast = parseSourceCode (file, File.ReadAllText(file)) Assert.True(hasAutoOpenAttributeInFile ast) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs index 39177fdcdc..6ce8395e3d 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs @@ -124,6 +124,7 @@ let time msg f a = result let mkGraph (files: FileWithAST array) = + // Implementation files backed by signatures should be excluded to construct the trie. let trieInput = files |> Array.filter (fun f -> @@ -150,11 +151,13 @@ let mkGraph (files: FileWithAST array) = let fileContent = fileContents.[file.Idx] let knownFiles = getFileNameBefore files file.Idx + // Process all entries of a file and query the trie when required to find the dependent files. let result = Seq.fold (processStateEntry queryTrie) (FileContentQueryState.Create file.Idx knownFiles) fileContent let allDependencies = if filesWithAutoOpen.Length > 0 then + // Automatically add all files that came before the current file that use the [] attribute. let autoOpenDependencies = set ([| 0 .. (file.Idx - 1) |].Intersect(filesWithAutoOpen)) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs index 06d71e63c3..0b105a3cdf 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/SampleData.fs @@ -4,6 +4,8 @@ open System.Collections.Generic open NUnit.Framework open ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution +// This file contains some hard coded data to easily debug the various aspects of the dependency resolution. + // Some helper DSL functions to construct the FileContentEntry items // This should again be mapped from the AST @@ -30,8 +32,7 @@ let prefIdent (lid: string) = let parts = lid.Split(".") Array.take (parts.Length - 1) parts |> List.ofArray |> PrefixedIdentifier -// Some hardcoded files processing, this was done by the naked eye and some regexes. - +// Some hardcoded files that reflect the file content of the first files in the Fantomas.Core project. let files = [| { @@ -615,7 +616,7 @@ let emptyHS () = HashSet(0) let indexOf name = Array.find (fun (fc: FileContent) -> fc.Name = name) files |> fun fc -> fc.Idx -// This should be constructed from the AST +// This should be constructed from the AST, again a hard coded subset of Fantomas.Core let fantomasCoreTrie: TrieNode = { Current = TrieNodeInfo.Root @@ -797,15 +798,3 @@ let ``ProcessOpenStatement full path match`` () = let dep = Seq.exactlyOne result.FoundDependencies Assert.AreEqual(indexOf "AstExtensions.fsi", dep) - -#if INTERACTIVE -open System.Text.RegularExpressions - -let fileContent = - System.IO.File.ReadAllText(@"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceParser.fs") - -Regex.Matches(fileContent, "(\\w)+(\\.(\\w)+)+") -|> Seq.cast -|> Seq.distinctBy (fun m -> m.Value) -|> Seq.iter (fun m -> printfn "prefIdent \"%s\"" m.Value) -#endif diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs index efcbebb0fe..2fd8c8367a 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/TrieMapping.fs @@ -228,30 +228,23 @@ let ``Fantomas Core trie`` () = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\ISourceTextExtensions.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\RangeHelpers.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fsi" - // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstExtensions.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\TriviaTypes.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Utils.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceParser.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fsi" - // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\AstTransformer.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Version.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Queue.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\FormatConfig.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fsi" - // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Defines.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fsi" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Trivia.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\SourceTransformer.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Context.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fsi" - // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodePrinter.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fsi" - // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatterImpl.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Validation.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fsi" - // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\Selection.fs" @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fsi" - // @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\CodeFormatter.fs" |] |> Array.mapi (fun idx file -> let ast = parseSourceCode (file, System.IO.File.ReadAllText(file)) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs index 34f6be530f..7f72c15b93 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/Types.fs @@ -4,7 +4,6 @@ open System.Collections.Generic open FSharp.Compiler.Syntax type File = string -type Files = Set type ModuleSegment = string type FileWithAST = diff --git a/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs b/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs index 35b2f843c6..8cabf26908 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TypedTreeGraph.fs @@ -21,15 +21,19 @@ let codebases = WorkDir = $@"{__SOURCE_DIRECTORY__}\.fcs_test\src\compiler" Path = $@"{__SOURCE_DIRECTORY__}\FCS.args.txt" } - // { - // WorkDir = $@"{__SOURCE_DIRECTORY__}\.fcs_test\tests\FSharp.Compiler.ComponentTests" - // Path = $@"{__SOURCE_DIRECTORY__}\ComponentTests.args.txt" - // } + { + WorkDir = $@"{__SOURCE_DIRECTORY__}\.fcs_test\tests\FSharp.Compiler.ComponentTests" + Path = $@"{__SOURCE_DIRECTORY__}\ComponentTests.args.txt" + } // Hard coded example ;) { WorkDir = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core" Path = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core\args.txt" } + { + WorkDir = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core.Tests" + Path = @"C:\Users\nojaf\Projects\main-fantomas\src\Fantomas.Core.Tests\args.txt" + } |] let checker = FSharpChecker.Create(keepAssemblyContents = true) @@ -227,6 +231,7 @@ let ``Create Graph from typed tree`` (code: Codebase) = let typedTreeMap = collectAllDeps graphFromTypedTree let heuristicMap = collectAllDeps graphFromHeuristic.Graph + /// Compare the found dependencies of a specified heuristic versus the dependencies found in the typed tree let compareDeps source fileName idx (depsFromHeuristic: Set) = let depsFromTypedTree = Map.find idx typedTreeMap From 09734e1a6f641cabb008ab6fed22a6903b8d74ab Mon Sep 17 00:00:00 2001 From: janusz Date: Sat, 19 Nov 2022 17:03:29 +0000 Subject: [PATCH 10/21] Fix build --- src/Compiler/Driver/ParseAndCheckInputs.fs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/Compiler/Driver/ParseAndCheckInputs.fs b/src/Compiler/Driver/ParseAndCheckInputs.fs index 0bfc5e7a57..5d1322efca 100644 --- a/src/Compiler/Driver/ParseAndCheckInputs.fs +++ b/src/Compiler/Driver/ParseAndCheckInputs.fs @@ -759,29 +759,17 @@ let ParseInputFilesInParallel (tcConfig: TcConfig, lexResourceManager, sourceFil for fileName in sourceFiles do checkInputFile tcConfig fileName - // Order files to be parsed by size (descending). The idea is to process big files first, - // so that near the end when only some nodes are still processing items, it's the smallest items, - // which should reduce the period of time where only some nodes are busy. - // This requires some empirical evidence. - let sourceFiles = - sourceFiles - |> List.mapi (fun i f -> i, f) - |> List.sortBy (fun (_i, f) -> -FileInfo(f).Length) - let sourceFiles = List.zip sourceFiles isLastCompiland UseMultipleDiagnosticLoggers (sourceFiles, delayLogger, None) (fun sourceFilesWithDelayLoggers -> sourceFilesWithDelayLoggers - |> ListParallel.map (fun (((idx, fileName), isLastCompiland), delayLogger) -> + |> ListParallel.map (fun ((fileName, isLastCompiland), delayLogger) -> let directoryName = Path.GetDirectoryName fileName let input = parseInputFileAux (tcConfig, lexResourceManager, fileName, (isLastCompiland, isExe), delayLogger, retryLocked) - idx, (input, directoryName)) - // Bring back index-based order - |> List.sortBy fst - |> List.map snd) + (input, directoryName))) let ParseInputFilesSequential (tcConfig: TcConfig, lexResourceManager, sourceFiles, diagnosticsLogger: DiagnosticsLogger, retryLocked) = let isLastCompiland, isExe = sourceFiles |> tcConfig.ComputeCanContainEntryPoint From bd8829839e5b2ba2d7e77fff67b07cf7802c771f Mon Sep 17 00:00:00 2001 From: janusz Date: Sat, 19 Nov 2022 21:56:46 +0000 Subject: [PATCH 11/21] Minor changes --- .../Code/TrieApproach/AutoOpenDetection.fs | 3 +- .../Code/TrieApproach/DependencyResolution.fs | 608 +++++++++--------- .../Tests/TestCompilation.fs | 15 +- 3 files changed, 321 insertions(+), 305 deletions(-) diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs index fbca7b37ba..2178acc8e5 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/AutoOpenDetection.fs @@ -16,7 +16,8 @@ let private autoOpenShapes = /// This isn't bullet proof but I wonder who would really alias this very core attribute. let isAutoOpenAttribute (attribute: SynAttribute) = match attribute.ArgExpr with - | SynExpr.Const(constant = SynConst.Unit _) + | SynExpr.Const(constant = SynConst.Unit) + | SynExpr.Const(constant = SynConst.Unit) | SynExpr.Const(constant = SynConst.String _) | SynExpr.Paren(expr = SynExpr.Const(constant = SynConst.String _)) -> let attributeName = diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs index 6ce8395e3d..abf6b3feb5 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs @@ -1,7 +1,9 @@ module ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution +module ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution open System.Linq open FSharp.Compiler.Syntax +open Internal.Utilities.Library.Extras // This is pseudo code of how we could restructure the trie code // My main benefit is that you can easily visually inspect if an identifier will match something in the trie @@ -177,7 +179,7 @@ open FSharp.Compiler.Service.Tests.Common let mkGraphAndReport files = let filesWithAST = files - |> Array.mapi (fun idx file -> + |> Array.Parallel.mapi (fun idx file -> { Idx = idx AST = parseSourceCode (file, System.IO.File.ReadAllText(file)) @@ -235,308 +237,308 @@ let ``Fantomas.Core for realzies`` () = let fcsFiles = [| - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSComp.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSIstrings.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\UtilsStrings.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\buildproperties.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.InternalsVisibleTo.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.AssemblyInfo.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\sformat.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\sformat.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\sr.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\sr.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ResizeArray.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ResizeArray.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\HashMultiMap.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\HashMultiMap.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\EditDistance.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\EditDistance.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\TaggedCollections.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\TaggedCollections.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\illib.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\illib.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\FileSystem.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\FileSystem.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ildiag.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ildiag.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\zmap.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\zmap.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\zset.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\zset.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\XmlAdapters.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\XmlAdapters.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\InternalCollections.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\InternalCollections.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\QueueList.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\QueueList.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\lib.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\lib.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ImmutableArray.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\ImmutableArray.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\rational.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\rational.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\PathMap.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\PathMap.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\RidHelpers.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\range.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Utilities\range.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\Logger.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\Logger.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\LanguageFeatures.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\LanguageFeatures.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticOptions.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticOptions.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\TextLayoutRender.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\TextLayoutRender.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticsLogger.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticsLogger.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticResolutionHints.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\DiagnosticResolutionHints.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\prim-lexing.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\prim-lexing.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\prim-parsing.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\prim-parsing.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\ReferenceResolver.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\ReferenceResolver.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\SimulatedMSBuildReferenceResolver.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\SimulatedMSBuildReferenceResolver.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\CompilerLocation.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\CompilerLocation.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\BuildGraph.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Facilities\BuildGraph.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\il.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\il.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilx.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilx.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilascii.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilascii.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\ilpars.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\illex.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilprint.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilprint.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilmorph.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilmorph.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilsign.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilsign.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilnativeres.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilnativeres.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilsupp.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilsupp.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilbinary.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilbinary.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilread.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilread.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilwritepdb.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilwritepdb.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilwrite.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilwrite.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilreflect.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\AbstractIL\ilreflect.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\PrettyNaming.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\PrettyNaming.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\UnicodeLexing.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\UnicodeLexing.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\XmlDoc.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\XmlDoc.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTrivia.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTrivia.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTree.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTree.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTreeOps.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\SyntaxTreeOps.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\ParseHelpers.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\ParseHelpers.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pppars.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pars.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\LexHelpers.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\LexHelpers.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pplex.fs" - @"C:\Users\nojaf\Projects\main-fsharp\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\\lex.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\LexFilter.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\SyntaxTree\LexFilter.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\tainted.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\tainted.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypeProviders.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypeProviders.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\QuotationPickler.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\QuotationPickler.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\CompilerGlobalState.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\CompilerGlobalState.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTree.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTree.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreeBasics.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreeBasics.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TcGlobals.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreeOps.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreeOps.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreePickle.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\TypedTree\TypedTreePickle.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\import.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\import.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\TypeHierarchy.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\TypeHierarchy.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\infos.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\infos.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AccessibilityLogic.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AccessibilityLogic.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AttributeChecking.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AttributeChecking.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\TypeRelations.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\TypeRelations.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\InfoReader.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\InfoReader.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\NicePrint.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\NicePrint.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AugmentWithHashCompare.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\AugmentWithHashCompare.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\NameResolution.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\NameResolution.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\SignatureConformance.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\SignatureConformance.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\MethodOverrides.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\MethodOverrides.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\MethodCalls.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\MethodCalls.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\PatternMatchCompilation.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\PatternMatchCompilation.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\ConstraintSolver.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\ConstraintSolver.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckFormatStrings.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckFormatStrings.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\FindUnsolved.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\FindUnsolved.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\QuotationTranslator.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\QuotationTranslator.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\PostInferenceChecks.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\PostInferenceChecks.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckBasics.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckBasics.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckExpressions.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckExpressions.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckPatterns.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckPatterns.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckComputationExpressions.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckComputationExpressions.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckIncrementalClasses.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckIncrementalClasses.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckDeclarations.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Checking\CheckDeclarations.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\Optimizer.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\Optimizer.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\DetupleArgs.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\DetupleArgs.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\InnerLambdasToTopLevelFuncs.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\InnerLambdasToTopLevelFuncs.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerCalls.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerCalls.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerSequences.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerSequences.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerComputedCollections.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerComputedCollections.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerStateMachines.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerStateMachines.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerLocalMutables.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Optimize\LowerLocalMutables.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\EraseClosures.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\EraseClosures.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\EraseUnions.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\EraseUnions.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\IlxGen.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\CodeGen\IlxGen.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\FxResolver.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\FxResolver.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/AssemblyResolveHandler.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/AssemblyResolveHandler.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/NativeDllResolveHandler.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/NativeDllResolveHandler.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/DependencyProvider.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\DependencyManager/DependencyProvider.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerConfig.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerConfig.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerImports.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerImports.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerDiagnostics.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerDiagnostics.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\ParseAndCheckInputs.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\ParseAndCheckInputs.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\ScriptClosure.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\ScriptClosure.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerOptions.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CompilerOptions.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\OptimizeInputs.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\OptimizeInputs.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\XmlDocFileWriter.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\XmlDocFileWriter.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\BinaryResourceFormats.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\BinaryResourceFormats.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\StaticLinking.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\StaticLinking.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CreateILModule.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\CreateILModule.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\fsc.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Driver\fsc.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\FSharpDiagnostic.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\FSharpDiagnostic.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\SymbolHelpers.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\SymbolHelpers.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\Symbols.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\Symbols.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\Exprs.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\Exprs.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\SymbolPatterns.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Symbols\SymbolPatterns.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\SemanticClassification.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\SemanticClassification.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ItemKey.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ItemKey.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\SemanticClassificationKey.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\SemanticClassificationKey.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpSource.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpSource.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\IncrementalBuild.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\IncrementalBuild.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceCompilerDiagnostics.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceCompilerDiagnostics.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceConstants.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceDeclarationLists.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceDeclarationLists.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceLexing.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceLexing.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParseTreeWalk.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParseTreeWalk.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceNavigation.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceNavigation.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParamInfoLocations.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParamInfoLocations.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpParseFileResults.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpParseFileResults.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParsedInputOps.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceParsedInputOps.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceAssemblyContent.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceAssemblyContent.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceXmlDocParser.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceXmlDocParser.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ExternalSymbol.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ExternalSymbol.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\QuickParse.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\QuickParse.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpCheckerResults.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\FSharpCheckerResults.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\service.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\service.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceInterfaceStubGenerator.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceInterfaceStubGenerator.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceStructure.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceStructure.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceAnalysis.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Service\ServiceAnalysis.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Interactive\ControlledExecution.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Interactive\fsi.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Interactive\fsi.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Legacy\LegacyMSBuildReferenceResolver.fsi" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Legacy\LegacyMSBuildReferenceResolver.fs" - @"C:\Users\nojaf\Projects\main-fsharp\src\Compiler\Legacy\LegacyHostedCompilerForTesting.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSComp.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSIstrings.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\UtilsStrings.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\buildproperties.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.InternalsVisibleTo.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\.NETStandard,Version=v2.0.AssemblyAttributes.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.AssemblyInfo.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\sformat.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\sformat.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\sr.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\sr.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ResizeArray.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ResizeArray.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\HashMultiMap.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\HashMultiMap.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\EditDistance.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\EditDistance.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\TaggedCollections.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\TaggedCollections.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\illib.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\illib.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\FileSystem.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\FileSystem.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ildiag.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ildiag.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\zmap.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\zmap.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\zset.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\zset.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\XmlAdapters.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\XmlAdapters.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\InternalCollections.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\InternalCollections.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\QueueList.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\QueueList.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\lib.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\lib.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ImmutableArray.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\ImmutableArray.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\rational.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\rational.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\PathMap.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\PathMap.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\RidHelpers.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\range.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Utilities\range.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\Logger.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\Logger.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\LanguageFeatures.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\LanguageFeatures.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticOptions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticOptions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\TextLayoutRender.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\TextLayoutRender.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticsLogger.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticsLogger.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticResolutionHints.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\DiagnosticResolutionHints.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\prim-lexing.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\prim-lexing.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\prim-parsing.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\prim-parsing.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\ReferenceResolver.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\ReferenceResolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\SimulatedMSBuildReferenceResolver.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\SimulatedMSBuildReferenceResolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\CompilerLocation.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\CompilerLocation.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\BuildGraph.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Facilities\BuildGraph.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\il.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\il.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilx.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilx.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilascii.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilascii.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\ilpars.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\illex.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilprint.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilprint.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilmorph.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilmorph.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilsign.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilsign.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilnativeres.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilnativeres.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilsupp.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilsupp.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilbinary.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilbinary.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilread.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilread.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilwritepdb.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilwritepdb.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilwrite.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilwrite.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilreflect.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\AbstractIL\ilreflect.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\PrettyNaming.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\PrettyNaming.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\UnicodeLexing.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\UnicodeLexing.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\XmlDoc.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\XmlDoc.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTrivia.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTrivia.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTree.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTree.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTreeOps.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\SyntaxTreeOps.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\ParseHelpers.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\ParseHelpers.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pppars.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pars.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\LexHelpers.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\LexHelpers.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\pplex.fs" + @"C:\projekty\fsharp\fsharp_main\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\\lex.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\LexFilter.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\SyntaxTree\LexFilter.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\tainted.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\tainted.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypeProviders.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypeProviders.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\QuotationPickler.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\QuotationPickler.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\CompilerGlobalState.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\CompilerGlobalState.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTree.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTree.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreeBasics.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreeBasics.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TcGlobals.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreeOps.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreeOps.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreePickle.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\TypedTree\TypedTreePickle.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\import.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\import.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\TypeHierarchy.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\TypeHierarchy.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\infos.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\infos.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AccessibilityLogic.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AccessibilityLogic.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AttributeChecking.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AttributeChecking.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\TypeRelations.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\TypeRelations.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\InfoReader.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\InfoReader.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\NicePrint.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\NicePrint.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AugmentWithHashCompare.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\AugmentWithHashCompare.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\NameResolution.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\NameResolution.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\SignatureConformance.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\SignatureConformance.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\MethodOverrides.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\MethodOverrides.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\MethodCalls.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\MethodCalls.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\PatternMatchCompilation.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\PatternMatchCompilation.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\ConstraintSolver.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\ConstraintSolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckFormatStrings.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckFormatStrings.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\FindUnsolved.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\FindUnsolved.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\QuotationTranslator.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\QuotationTranslator.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\PostInferenceChecks.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\PostInferenceChecks.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckBasics.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckBasics.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckExpressions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckExpressions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckPatterns.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckPatterns.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckComputationExpressions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckComputationExpressions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckIncrementalClasses.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckIncrementalClasses.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckDeclarations.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Checking\CheckDeclarations.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\Optimizer.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\Optimizer.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\DetupleArgs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\DetupleArgs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\InnerLambdasToTopLevelFuncs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\InnerLambdasToTopLevelFuncs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerCalls.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerCalls.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerSequences.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerSequences.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerComputedCollections.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerComputedCollections.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerStateMachines.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerStateMachines.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerLocalMutables.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Optimize\LowerLocalMutables.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\EraseClosures.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\EraseClosures.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\EraseUnions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\EraseUnions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\IlxGen.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\CodeGen\IlxGen.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\FxResolver.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\FxResolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/AssemblyResolveHandler.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/AssemblyResolveHandler.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/NativeDllResolveHandler.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/NativeDllResolveHandler.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/DependencyProvider.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\DependencyManager/DependencyProvider.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerConfig.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerConfig.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerImports.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerImports.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerDiagnostics.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerDiagnostics.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\ParseAndCheckInputs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\ParseAndCheckInputs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\ScriptClosure.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\ScriptClosure.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerOptions.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CompilerOptions.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\OptimizeInputs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\OptimizeInputs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\XmlDocFileWriter.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\XmlDocFileWriter.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\BinaryResourceFormats.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\BinaryResourceFormats.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\StaticLinking.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\StaticLinking.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CreateILModule.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\CreateILModule.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\fsc.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Driver\fsc.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\FSharpDiagnostic.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\FSharpDiagnostic.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\SymbolHelpers.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\SymbolHelpers.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\Symbols.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\Symbols.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\Exprs.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\Exprs.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\SymbolPatterns.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Symbols\SymbolPatterns.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\SemanticClassification.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\SemanticClassification.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ItemKey.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ItemKey.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\SemanticClassificationKey.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\SemanticClassificationKey.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpSource.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpSource.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\IncrementalBuild.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\IncrementalBuild.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceCompilerDiagnostics.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceCompilerDiagnostics.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceConstants.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceDeclarationLists.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceDeclarationLists.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceLexing.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceLexing.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParseTreeWalk.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParseTreeWalk.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceNavigation.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceNavigation.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParamInfoLocations.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParamInfoLocations.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpParseFileResults.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpParseFileResults.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParsedInputOps.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceParsedInputOps.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceAssemblyContent.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceAssemblyContent.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceXmlDocParser.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceXmlDocParser.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ExternalSymbol.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ExternalSymbol.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\QuickParse.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\QuickParse.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpCheckerResults.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\FSharpCheckerResults.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\service.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\service.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceInterfaceStubGenerator.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceInterfaceStubGenerator.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceStructure.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceStructure.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceAnalysis.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Service\ServiceAnalysis.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Interactive\ControlledExecution.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Interactive\fsi.fsi" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Interactive\fsi.fs" + // @"C:\projekty\fsharp\fsharp_main\src\Compiler\Legacy\LegacyMSBuildReferenceResolver.fsi" + // @"C:\projekty\fsharp\fsharp_main\src\Compiler\Legacy\LegacyMSBuildReferenceResolver.fs" + @"C:\projekty\fsharp\fsharp_main\src\Compiler\Legacy\LegacyHostedCompilerForTesting.fs" |] [] diff --git a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs index 8045b0de5d..988865e381 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs @@ -118,11 +118,24 @@ let b = 1 "A.fs", """ namespace A +module A1 = +module A1 = + let x = 3 + type X = X of int + let y = X 5 + let foo () = 5 + let inline bar () = 7 + module B2 = + let a = "lalala" """ "B.fs", """ module B open A +let z = A.A1.x +let y = A.A1.y +let g = A.A1.x + 2 +let h = A.A1.bar() """ ] |> FProject.Make CompileOutput.Library @@ -221,7 +234,7 @@ let withMethod (method: Method) (cu: CompilationUnit) : CompilationUnit = | CompilationUnit.FS cs -> FS { cs with - Options = cs.Options @ (methodOptions method) + Options = cs.Options @ (methodOptions method) @ ["--optimize-"] } | cu -> cu From a5b6d888af0f8782feec466551afd9e055433983 Mon Sep 17 00:00:00 2001 From: janusz Date: Sun, 20 Nov 2022 00:42:00 +0000 Subject: [PATCH 12/21] Seems to work for small examples. --- src/Compiler/Driver/OptimizeInputs.fs | 512 +++++++++++++++--- src/Compiler/Driver/OptimizeInputs.fsi | 2 + src/Compiler/Driver/Parallel.fs | 54 ++ src/Compiler/FSharp.Compiler.Service.fsproj | 1 + tests/FSharp.Test.Utilities/Compiler.fs | 2 +- tests/FSharp.Test.Utilities/TestFramework.fs | 2 +- .../Code/Parallel.fs | 1 + .../Code/TrieApproach/DependencyResolution.fs | 1 - .../Tests/TestCompilation.fs | 15 +- .../Tests/TestCompilationFromCmdlineArgs.fs | 2 +- 10 files changed, 510 insertions(+), 82 deletions(-) create mode 100644 src/Compiler/Driver/Parallel.fs diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index 0696758780..8b9d1e32f1 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -2,7 +2,11 @@ module internal FSharp.Compiler.OptimizeInputs +open System.Collections.Generic +open System.Diagnostics open System.IO +open System.Threading +open FSharp.Compiler.Optimizer open Internal.Utilities.Library open FSharp.Compiler open FSharp.Compiler.AbstractIL.IL @@ -48,6 +52,232 @@ let GetInitialOptimizationEnv (tcImports: TcImports, tcGlobals: TcGlobals) = let optEnv = List.fold (AddExternalCcuToOptimizationEnv tcGlobals) optEnv ccuinfos optEnv +type OptimizeDuringCodeGen = bool -> Expr -> Expr +type OptimizeRes = + (IncrementalOptimizationEnv * CheckedImplFile * ImplFileOptimizationInfo * SignatureHidingInfo) * OptimizeDuringCodeGen + +type Optimize = + OptimizationSettings * + CcuThunk * + TcGlobals * + ConstraintSolver.TcValF * + Import.ImportMap * + IncrementalOptimizationEnv * + bool * + bool * + bool * + SignatureHidingInfo * + CheckedImplFile -> + OptimizeRes + +type PhaseInputs = IncrementalOptimizationEnv * SignatureHidingInfo * CheckedImplFile + +type Phase1Inputs = PhaseInputs +type Phase1Res = OptimizeRes +type Phase1Fun = Phase1Inputs -> Phase1Res + +type Phase2Inputs = PhaseInputs +type Phase2Res = IncrementalOptimizationEnv * CheckedImplFile +type Phase2Fun = Phase2Inputs -> Phase2Res + +type Phase3Inputs = PhaseInputs +type Phase3Res = IncrementalOptimizationEnv * CheckedImplFile +type Phase3Fun = Phase3Inputs -> Phase3Res + +type FileResultsComplete = + { + Phase1: Phase1Res + Phase2: Phase2Res + Phase3: Phase3Res + } +type CollectorInputs = FileResultsComplete[] +type CollectorOutputs = (CheckedImplFileAfterOptimization * ImplFileOptimizationInfo)[] * IncrementalOptimizationEnv + +let collectResults (inputs: CollectorInputs) : CollectorOutputs = + let files = + inputs + |> Array.map (fun {Phase1 = phase1; Phase2 = _phase2; Phase3 = phase3} -> + let (_, _, implFileOptData, _), optimizeDuringCodeGen = phase1 + let _, implFile = phase3 + let implFile = + { + ImplFile = implFile + OptimizeDuringCodeGen = optimizeDuringCodeGen + } + implFile, implFileOptData + ) + + let lastFilePhase1Env = + inputs + |> Array.last + |> fun {Phase1 = phase1} -> + let (optEnvPhase1, _, _, _), _ = phase1 + optEnvPhase1 + + files, lastFilePhase1Env + +type Phase = + | Phase1 + | Phase2 + | Phase3 + +type FilePhaseFuncs = Phase1Fun * Phase2Fun * Phase3Fun +type FileResults = + { + mutable Phase1: Phase1Res option + mutable Phase2: Phase2Res option + mutable Phase3: Phase3Res option + } + with static member Empty = + { + Phase1 = None + Phase2 = None + Phase3 = None + } + +type WorkItem = + | Phase1 of Phase1Inputs + | Phase2 of Phase2Inputs + | Phase3 of Phase3Inputs + +type Idx = int +type Node = + { + Idx: Idx + Phase: Phase + } + with override this.ToString() = $"[{this.Idx}-{this.Phase}]" + + +let getPhase1Res (p: FileResults) = + p.Phase1 + |> Option.get + |> fun ((env, _, _, hidden), _) -> env, hidden + +let getPhase2Res (p: FileResults) = + p.Phase2 + |> Option.get + |> fun (env, _) -> env + +let getPhase3Res (p: FileResults) = + p.Phase3 + |> Option.get + |> fun (env, _) -> env + +let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFuncs) (files: CheckedImplFile[]) : CollectorOutputs = + // Schedule File1-Phase1 + let firstNode = { Idx = 0; Phase = Phase.Phase1 } + + let results = + files + |> Array.map (fun _ -> FileResults.Empty) + + let _lock = obj() + + let visited = HashSet() + + let worker ({Idx = idx; Phase = phase} as node : Node) : Node[] = + let notPreviouslyVisited = + lock (_lock) (fun () -> + visited.Add node + ) + if notPreviouslyVisited = false then [||] + else + let res = results[idx] + let file = files[idx] + let previous = if idx > 0 then Some results[idx-1] else None + let hidingInfo0 = SignatureHidingInfo.Empty + + let getPhase1Res (p: FileResults) = + p.Phase1 + |> Option.get + |> fun ((env, _, _, hidden), _) -> env, hidden + + match phase with + | Phase.Phase1 -> + // take env from previous file + let env, hidingInfo = + previous + |> Option.map getPhase1Res + |> Option.defaultValue (env0, hidingInfo0) + let inputs = env, hidingInfo, file + let phase1Res = phase1 inputs + res.Phase1 <- Some phase1Res + + // Schedule Phase2 + let phase2Node = { Idx = idx; Phase = Phase.Phase2 } + seq { + yield phase2Node + if idx < files.Length-1 then yield { Idx = idx + 1; Phase = Phase.Phase1 } + } + |> Seq.toArray + + | Phase.Phase2 -> + // take env from previous file + let env = + previous + |> Option.map getPhase2Res + |> Option.defaultValue env0 + let hidingInfo = + res + |> getPhase1Res + |> snd + let inputs = env, hidingInfo, file + let phase2Res = phase2 inputs + res.Phase2 <- Some phase2Res + + seq { + // Schedule Phase3 + let phase3Node = { Idx = idx; Phase = Phase.Phase3 } + yield phase3Node + // Schedule Phase2 for the next file if it exists + if idx < files.Length-1 then yield { Idx = idx + 1; Phase = Phase.Phase2 } + } + |> Seq.toArray + + | Phase.Phase3 -> + // take env from previous file + let env = + previous + |> Option.map getPhase3Res + |> Option.defaultValue env0 + let hidingInfo = + previous + |> Option.map getPhase1Res + |> Option.map snd + |> Option.defaultValue hidingInfo0 + let inputs = env, hidingInfo, file + let phase3Res = phase3 inputs + res.Phase3 <- Some phase3Res + + seq { + // Schedule Phase3 for the next file if it exists + if idx < files.Length-1 then yield { Idx = idx + 1; Phase = Phase.Phase3 } + } + |> Seq.toArray + + ParallelTypeCheckingTests.Parallel.processInParallel + [|firstNode|] + worker + 10 + (fun _ -> visited.Count >= files.Length * 3) + (CancellationToken.None) + (fun node -> node.ToString()) + + Debug.Assert(visited.Count = files.Length * 3) + + let results = + results + |> Array.mapi (fun i {Phase1 = phase1; Phase2 = phase2; Phase3 = phase3} -> + match phase1, phase2, phase3 with + | Some phase1, Some phase2, Some phase3 -> {FileResultsComplete.Phase1 = phase1; Phase2 = phase2; Phase3 = phase3} + | _ -> failwith $"Unexpected lack of results for file [{i}]" + ) + let collected = results |> collectResults + collected + +let mutable UseParallelOptimizer: bool = false + let ApplyAllOptimizations ( tcConfig: TcConfig, @@ -90,20 +320,53 @@ let ApplyAllOptimizations reportingPhase = true } - let results, (optEnvFirstLoop, _, _, _) = - ((optEnv0, optEnv0, optEnv0, SignatureHidingInfo.Empty), implFiles) - - ||> List.mapFold (fun (optEnvFirstLoop, optEnvExtraLoop, optEnvFinalSimplify, hidden) implFile -> - - //ReportTime tcConfig ("Initial simplify") - let (optEnvFirstLoop, implFile, implFileOptData, hidden), optimizeDuringCodeGen = + + let env0 = optEnv0 + + let phase1 (env: IncrementalOptimizationEnv, hidden: SignatureHidingInfo, implFile: CheckedImplFile) : Phase1Res = + //ReportTime tcConfig ("Initial simplify") + Optimizer.OptimizeImplFile( + optSettings, + ccu, + tcGlobals, + tcVal, + importMap, + env, + isIncrementalFragment, + tcConfig.fsiMultiAssemblyEmit, + tcConfig.emitTailcalls, + hidden, + implFile + ) + + let phase2 (env: IncrementalOptimizationEnv, hidden: SignatureHidingInfo, implFile: CheckedImplFile) : Phase2Res = + let implFile = LowerLocalMutables.TransformImplFile tcGlobals importMap implFile + + // Only do this on the first pass! + let optSettings = + { optSettings with + abstractBigTargets = false + reportingPhase = false + } +// TODO Uncomment +// #if DEBUG +// if tcConfig.showOptimizationData then +// dprintf +// "Optimization implFileOptData:\n%s\n" +// (LayoutRender.showL (Display.squashTo 192 (Optimizer.moduleInfoL tcGlobals implFileOptData))) +// #endif + + if tcConfig.extraOptimizationIterations > 0 then + + //ReportTime tcConfig ("Extra simplification loop") + let (optEnvExtraLoop, implFile, _, _), _ = Optimizer.OptimizeImplFile( optSettings, ccu, tcGlobals, tcVal, importMap, - optEnvFirstLoop, + env, isIncrementalFragment, tcConfig.fsiMultiAssemblyEmit, tcConfig.emitTailcalls, @@ -111,33 +374,82 @@ let ApplyAllOptimizations implFile ) - let implFile = LowerLocalMutables.TransformImplFile tcGlobals importMap implFile + //PrintWholeAssemblyImplementation tcConfig outfile (sprintf "extra-loop-%d" n) implFile + optEnvExtraLoop, implFile + else + env, implFile + + let phase3 (env: IncrementalOptimizationEnv, hidden: SignatureHidingInfo, implFile: CheckedImplFile) : Phase3Res = + // Only do this on the first pass! + let optSettings = + { optSettings with + abstractBigTargets = false + reportingPhase = false + } + + let implFile = + if tcConfig.doDetuple then + //ReportTime tcConfig ("Detupled optimization") + let implFile = implFile |> Detuple.DetupleImplFile ccu tcGlobals + //PrintWholeAssemblyImplementation tcConfig outfile "post-detuple" implFile + implFile + else + implFile + + let implFile = + if tcConfig.doTLR then + implFile + |> InnerLambdasToTopLevelFuncs.MakeTopLevelRepresentationDecisions ccu tcGlobals + else + implFile + + let implFile = LowerCalls.LowerImplFile tcGlobals implFile + + if tcConfig.doFinalSimplify then + + //ReportTime tcConfig ("Final simplify pass") + let (optEnvFinalSimplify, implFile, _, _), _ = + Optimizer.OptimizeImplFile( + optSettings, + ccu, + tcGlobals, + tcVal, + importMap, + env, + isIncrementalFragment, + tcConfig.fsiMultiAssemblyEmit, + tcConfig.emitTailcalls, + hidden, + implFile + ) - // Only do this on the first pass! - let optSettings = - { optSettings with - abstractBigTargets = false - reportingPhase = false - } -#if DEBUG - if tcConfig.showOptimizationData then - dprintf - "Optimization implFileOptData:\n%s\n" - (LayoutRender.showL (Display.squashTo 192 (Optimizer.moduleInfoL tcGlobals implFileOptData))) -#endif + //PrintWholeAssemblyImplementation tcConfig outfile "post-rec-opt" implFile + optEnvFinalSimplify, implFile + else + env, implFile + + let results, optEnvFirstLoop = + if UseParallelOptimizer then + let a, b = + go env0 (phase1, phase2, phase3) (implFiles |> List.toArray) + a |> Array.toList, b + else + let results, (optEnvFirstLoop, _, _, _) = + ((optEnv0, optEnv0, optEnv0, SignatureHidingInfo.Empty), implFiles) - let implFile, optEnvExtraLoop = - if tcConfig.extraOptimizationIterations > 0 then + ||> List.mapFold (fun (optEnvFirstLoop, optEnvExtraLoop, optEnvFinalSimplify, hidden) implFile -> - //ReportTime tcConfig ("Extra simplification loop") - let (optEnvExtraLoop, implFile, _, _), _ = + // Phase 1 + + //ReportTime tcConfig ("Initial simplify") + let (optEnvFirstLoop, implFile, implFileOptData, hidden), optimizeDuringCodeGen = Optimizer.OptimizeImplFile( optSettings, ccu, tcGlobals, tcVal, importMap, - optEnvExtraLoop, + optEnvFirstLoop, isIncrementalFragment, tcConfig.fsiMultiAssemblyEmit, tcConfig.emitTailcalls, @@ -145,60 +457,108 @@ let ApplyAllOptimizations implFile ) - //PrintWholeAssemblyImplementation tcConfig outfile (sprintf "extra-loop-%d" n) implFile - implFile, optEnvExtraLoop - else - implFile, optEnvExtraLoop - let implFile = - if tcConfig.doDetuple then - //ReportTime tcConfig ("Detupled optimization") - let implFile = implFile |> Detuple.DetupleImplFile ccu tcGlobals - //PrintWholeAssemblyImplementation tcConfig outfile "post-detuple" implFile - implFile - else - implFile - - let implFile = - if tcConfig.doTLR then - implFile - |> InnerLambdasToTopLevelFuncs.MakeTopLevelRepresentationDecisions ccu tcGlobals - else - implFile - - let implFile = LowerCalls.LowerImplFile tcGlobals implFile - - let implFile, optEnvFinalSimplify = - if tcConfig.doFinalSimplify then - - //ReportTime tcConfig ("Final simplify pass") - let (optEnvFinalSimplify, implFile, _, _), _ = - Optimizer.OptimizeImplFile( - optSettings, - ccu, - tcGlobals, - tcVal, - importMap, - optEnvFinalSimplify, - isIncrementalFragment, - tcConfig.fsiMultiAssemblyEmit, - tcConfig.emitTailcalls, - hidden, + // Phase 2 + + let implFile = LowerLocalMutables.TransformImplFile tcGlobals importMap implFile + + // Only do this on the first pass! + let optSettings = + { optSettings with + abstractBigTargets = false + reportingPhase = false + } + #if DEBUG + if tcConfig.showOptimizationData then + dprintf + "Optimization implFileOptData:\n%s\n" + (LayoutRender.showL (Display.squashTo 192 (Optimizer.moduleInfoL tcGlobals implFileOptData))) + #endif + + let implFile, optEnvExtraLoop = + if tcConfig.extraOptimizationIterations > 0 then + + //ReportTime tcConfig ("Extra simplification loop") + let (optEnvExtraLoop, implFile, _, _), _ = + Optimizer.OptimizeImplFile( + optSettings, + ccu, + tcGlobals, + tcVal, + importMap, + optEnvExtraLoop, + isIncrementalFragment, + tcConfig.fsiMultiAssemblyEmit, + tcConfig.emitTailcalls, + hidden, + implFile + ) + + //PrintWholeAssemblyImplementation tcConfig outfile (sprintf "extra-loop-%d" n) implFile + implFile, optEnvExtraLoop + else + implFile, optEnvExtraLoop + + + // Phase 3 + + // Only do this on the first pass! + let optSettings = + { optSettings with + abstractBigTargets = false + reportingPhase = false + } + + let implFile = + if tcConfig.doDetuple then + //ReportTime tcConfig ("Detupled optimization") + let implFile = implFile |> Detuple.DetupleImplFile ccu tcGlobals + //PrintWholeAssemblyImplementation tcConfig outfile "post-detuple" implFile + implFile + else implFile - ) - - //PrintWholeAssemblyImplementation tcConfig outfile "post-rec-opt" implFile - implFile, optEnvFinalSimplify - else - implFile, optEnvFinalSimplify - let implFile = - { - ImplFile = implFile - OptimizeDuringCodeGen = optimizeDuringCodeGen - } + let implFile = + if tcConfig.doTLR then + implFile + |> InnerLambdasToTopLevelFuncs.MakeTopLevelRepresentationDecisions ccu tcGlobals + else + implFile - (implFile, implFileOptData), (optEnvFirstLoop, optEnvExtraLoop, optEnvFinalSimplify, hidden)) + let implFile = LowerCalls.LowerImplFile tcGlobals implFile + + let implFile, optEnvFinalSimplify = + if tcConfig.doFinalSimplify then + + //ReportTime tcConfig ("Final simplify pass") + let (optEnvFinalSimplify, implFile, _, _), _ = + Optimizer.OptimizeImplFile( + optSettings, + ccu, + tcGlobals, + tcVal, + importMap, + optEnvFinalSimplify, + isIncrementalFragment, + tcConfig.fsiMultiAssemblyEmit, + tcConfig.emitTailcalls, + hidden, + implFile + ) + + //PrintWholeAssemblyImplementation tcConfig outfile "post-rec-opt" implFile + implFile, optEnvFinalSimplify + else + implFile, optEnvFinalSimplify + + let implFile = + { + ImplFile = implFile + OptimizeDuringCodeGen = optimizeDuringCodeGen + } + + (implFile, implFileOptData), (optEnvFirstLoop, optEnvExtraLoop, optEnvFinalSimplify, hidden)) + results, optEnvFirstLoop let implFiles, implFileOptDatas = List.unzip results let assemblyOptData = Optimizer.UnionOptimizationInfos implFileOptDatas diff --git a/src/Compiler/Driver/OptimizeInputs.fsi b/src/Compiler/Driver/OptimizeInputs.fsi index d5c731ba05..33881ae6cb 100644 --- a/src/Compiler/Driver/OptimizeInputs.fsi +++ b/src/Compiler/Driver/OptimizeInputs.fsi @@ -49,3 +49,5 @@ val GenerateIlxCode: val NormalizeAssemblyRefs: CompilationThreadToken * ILGlobals * TcImports -> (ILScopeRef -> ILScopeRef) val GetGeneratedILModuleName: CompilerTarget -> string -> string + +val mutable UseParallelOptimizer: bool \ No newline at end of file diff --git a/src/Compiler/Driver/Parallel.fs b/src/Compiler/Driver/Parallel.fs new file mode 100644 index 0000000000..dbe9394a63 --- /dev/null +++ b/src/Compiler/Driver/Parallel.fs @@ -0,0 +1,54 @@ +module ParallelTypeCheckingTests.Parallel + +#nowarn "1182" + +open System +open System.Collections.Concurrent +open System.Threading + +// TODO Could replace with MailboxProcessor+Tasks/Asyncs instead of BlockingCollection + Threads +// See http://www.fssnip.net/nX/title/Limit-degree-of-parallelism-using-an-agent +/// Process items in parallel, allow more work to be scheduled as a result of finished work, +/// limit parallelisation to 'parallelism' threads +let processInParallel + (firstItems: 'Item[]) + (work: 'Item -> 'Item[]) + (parallelism: int) + (shouldStop: int -> bool) + (ct: CancellationToken) + (_itemToString: 'Item -> string) + : unit = + let bc = new BlockingCollection<'Item>() + firstItems |> Array.iter bc.Add + let processedCountLock = Object() + let mutable processedCount = 0 + + let processItem item = + printfn $"[{Thread.CurrentThread.ManagedThreadId}] Processing {_itemToString item}" + let toSchedule = work item + printfn $"[{Thread.CurrentThread.ManagedThreadId}] Finished {_itemToString item}" + + let processedCount = + lock processedCountLock (fun () -> + processedCount <- processedCount + 1 + processedCount) + + let toScheduleString = + toSchedule |> Array.map _itemToString |> (fun names -> String.Join(", ", names)) + + printfn $"[{Thread.CurrentThread.ManagedThreadId}] Scheduling {toSchedule.Length} items: {toScheduleString}" + toSchedule |> Array.iter bc.Add + processedCount + + // TODO Could avoid workers with some semaphores + let workerWork () : unit = + for node in bc.GetConsumingEnumerable(ct) do + if not ct.IsCancellationRequested then // improve + let processedCount = processItem node + + if shouldStop processedCount then + bc.CompleteAdding() + + // TODO Do we need to handle cancellation given that workers do it already? + Array.Parallel.map workerWork (Array.init parallelism (fun _ -> ())) |> ignore + diff --git a/src/Compiler/FSharp.Compiler.Service.fsproj b/src/Compiler/FSharp.Compiler.Service.fsproj index f479840dfb..73624e0c27 100644 --- a/src/Compiler/FSharp.Compiler.Service.fsproj +++ b/src/Compiler/FSharp.Compiler.Service.fsproj @@ -387,6 +387,7 @@ + diff --git a/tests/FSharp.Test.Utilities/Compiler.fs b/tests/FSharp.Test.Utilities/Compiler.fs index bf76999393..f3cb5f6219 100644 --- a/tests/FSharp.Test.Utilities/Compiler.fs +++ b/tests/FSharp.Test.Utilities/Compiler.fs @@ -404,7 +404,7 @@ module rec Compiler = withOptionsHelper [ $"--nowarn:{warning}" ] "withNoWarn is only supported for F#" cUnit let withNoOptimize (cUnit: CompilationUnit) : CompilationUnit = - withOptionsHelper [ "--optimize-" ] "withNoOptimize is only supported for F#" cUnit + withOptionsHelper [ "--optimize+" ] "withNoOptimize is only supported for F#" cUnit let withOptimize (cUnit: CompilationUnit) : CompilationUnit = withOptionsHelper [ "--optimize+" ] "withOptimize is only supported for F#" cUnit diff --git a/tests/FSharp.Test.Utilities/TestFramework.fs b/tests/FSharp.Test.Utilities/TestFramework.fs index 78f2ea8193..0468d3ee66 100644 --- a/tests/FSharp.Test.Utilities/TestFramework.fs +++ b/tests/FSharp.Test.Utilities/TestFramework.fs @@ -437,7 +437,7 @@ let envVars () = let initializeSuite () = #if DEBUG - let configurationName = "Debug" + let configurationName = "Release"// TODO "Debug" #else let configurationName = "Release" #endif diff --git a/tests/ParallelTypeCheckingTests/Code/Parallel.fs b/tests/ParallelTypeCheckingTests/Code/Parallel.fs index 45e6379b8d..33df5fb267 100644 --- a/tests/ParallelTypeCheckingTests/Code/Parallel.fs +++ b/tests/ParallelTypeCheckingTests/Code/Parallel.fs @@ -50,3 +50,4 @@ let processInParallel // TODO Do we need to handle cancellation given that workers do it already? Array.Parallel.map workerWork (Array.init parallelism (fun _ -> ())) |> ignore + diff --git a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs index abf6b3feb5..378a702cb3 100644 --- a/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs +++ b/tests/ParallelTypeCheckingTests/Code/TrieApproach/DependencyResolution.fs @@ -1,5 +1,4 @@ module ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution -module ParallelTypeCheckingTests.Code.TrieApproach.DependencyResolution open System.Linq open FSharp.Compiler.Syntax diff --git a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs index 988865e381..93f43a9ef6 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs @@ -4,6 +4,7 @@ open FSharp.Test open FSharp.Test.Compiler open NUnit.Framework open ParallelTypeCheckingTests.TestUtils +open FSharp.Compiler type FProject = { @@ -118,7 +119,6 @@ let b = 1 "A.fs", """ namespace A -module A1 = module A1 = let x = 3 type X = X of int @@ -234,7 +234,7 @@ let withMethod (method: Method) (cu: CompilationUnit) : CompilationUnit = | CompilationUnit.FS cs -> FS { cs with - Options = cs.Options @ (methodOptions method) @ ["--optimize-"] + Options = cs.Options @ (methodOptions method) @ ["--optimize+"] } | cu -> cu @@ -253,8 +253,19 @@ let compileAValidProject (x: Case) = let codebases = Codebases.all + [] let ``Compile a valid project using graph-based type-checking`` (project: FProject) = + global.FSharp.Compiler.OptimizeInputs.UseParallelOptimizer <- false + compileAValidProject + { + Method = Method.Graph + Project = project + } + +[] +let ``Compile a valid project using graph-based type-checking, parallel opt`` (project: FProject) = + global.FSharp.Compiler.OptimizeInputs.UseParallelOptimizer <- true compileAValidProject { Method = Method.Graph diff --git a/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs b/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs index 6bc069f0fe..109e47e64c 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs @@ -75,7 +75,7 @@ let internal TestCompilerFromArgs (config: Args) : unit = try let args = setupParsed config - let exit: int = CommandLineMain.mainAux (args, true, Some exiter) + let exit: int = CommandLineMain.mainAux (args, false, Some exiter) // TODO reset to true Assert.That(exit, Is.Zero) finally Environment.CurrentDirectory <- oldWorkDir From 0c52ca67a2b9196a01b8920ed78e24ed5b3e6895 Mon Sep 17 00:00:00 2001 From: janusz Date: Sun, 20 Nov 2022 01:11:23 +0000 Subject: [PATCH 13/21] changes --- src/Compiler/Driver/OptimizeInputs.fs | 36 ++++++++++++++++++---- tests/ParallelTypeCheckingTests/Program.fs | 18 +++++++++-- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index 8b9d1e32f1..19ce7f2402 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -128,12 +128,18 @@ type FileResults = mutable Phase2: Phase2Res option mutable Phase3: Phase3Res option } - with static member Empty = - { - Phase1 = None - Phase2 = None - Phase3 = None - } + with + member this.HasResult (phase: Phase) = + match phase with + | Phase.Phase1 -> this.Phase1 |> Option.isSome + | Phase.Phase2 -> this.Phase2 |> Option.isSome + | Phase.Phase3 -> this.Phase3 |> Option.isSome + static member Empty = + { + Phase1 = None + Phase2 = None + Phase3 = None + } type WorkItem = | Phase1 of Phase1Inputs @@ -173,6 +179,21 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu |> Array.map (fun _ -> FileResults.Empty) let _lock = obj() + let nodeCanBeProcessed ({Idx = idx; Phase = phase}) : bool = + lock (_lock) (fun () -> + let previousFileReady = + if idx = 0 then true else results[idx-1].HasResult phase + let previousPhase = + match phase with + | Phase.Phase1 -> None + | Phase.Phase2 -> Some Phase.Phase1 + | Phase.Phase3 -> Some Phase.Phase2 + let previousPhaseReady = + match previousPhase with + | Some previousPhase -> results[idx].HasResult previousPhase + | None -> true + previousFileReady && previousPhaseReady + ) let visited = HashSet() @@ -255,6 +276,9 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu if idx < files.Length-1 then yield { Idx = idx + 1; Phase = Phase.Phase3 } } |> Seq.toArray + |> fun nodes -> + nodes + |> Array.filter nodeCanBeProcessed ParallelTypeCheckingTests.Parallel.processInParallel [|firstNode|] diff --git a/tests/ParallelTypeCheckingTests/Program.fs b/tests/ParallelTypeCheckingTests/Program.fs index 9e0c9dd961..517e4403a1 100644 --- a/tests/ParallelTypeCheckingTests/Program.fs +++ b/tests/ParallelTypeCheckingTests/Program.fs @@ -2,7 +2,9 @@ #nowarn "1182" +open FSharp.Compiler open FSharp.Compiler.CompilerConfig +open ParallelTypeCheckingTests.TestCompilation open ParallelTypeCheckingTests.TestUtils let _parse (argv: string[]) : Args = @@ -27,9 +29,19 @@ let _parse (argv: string[]) : Args = WorkingDir = workingDir } +open ParallelTypeCheckingTests.TestCompilationFromCmdlineArgs [] let main _argv = - let args = _parse _argv - let args = { args with LineLimit = None } - TestCompilationFromCmdlineArgs.TestCompilerFromArgs args + OptimizeInputs.UseParallelOptimizer <- true + // let args = _parse _argv + // let args = { args with LineLimit = None } + let componentTests = codebases[1] + let config = codebaseToConfig componentTests Method.ParallelCheckingOfBackedImplFiles + TestCompilerFromArgs config + compileAValidProject + { + Method = Method.Sequential + Project = Codebases.dependentSignatures + } 0 + From 6953e23368fda803a4e1bae20f47dce02aceb9ab Mon Sep 17 00:00:00 2001 From: janusz Date: Sun, 20 Nov 2022 01:46:07 +0000 Subject: [PATCH 14/21] PoC - fails on FCS, works on ComponentTests. --- tests/ParallelTypeCheckingTests/Program.fs | 13 +++++-------- .../Tests/ComponentTests.args.txt | 2 +- tests/ParallelTypeCheckingTests/Tests/FCS.args.txt | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/ParallelTypeCheckingTests/Program.fs b/tests/ParallelTypeCheckingTests/Program.fs index 517e4403a1..190a046bec 100644 --- a/tests/ParallelTypeCheckingTests/Program.fs +++ b/tests/ParallelTypeCheckingTests/Program.fs @@ -32,16 +32,13 @@ let _parse (argv: string[]) : Args = open ParallelTypeCheckingTests.TestCompilationFromCmdlineArgs [] let main _argv = - OptimizeInputs.UseParallelOptimizer <- true + ParseAndCheckInputs.CheckMultipleInputsUsingGraphMode <- + ParallelTypeChecking.CheckMultipleInputsInParallel + OptimizeInputs.UseParallelOptimizer <- bool.Parse(_argv[0]) // let args = _parse _argv // let args = { args with LineLimit = None } - let componentTests = codebases[1] - let config = codebaseToConfig componentTests Method.ParallelCheckingOfBackedImplFiles + let componentTests = codebases[System.Int32.Parse(_argv[1])] + let config = codebaseToConfig componentTests Method.Graph TestCompilerFromArgs config - compileAValidProject - { - Method = Method.Sequential - Project = Codebases.dependentSignatures - } 0 diff --git a/tests/ParallelTypeCheckingTests/Tests/ComponentTests.args.txt b/tests/ParallelTypeCheckingTests/Tests/ComponentTests.args.txt index 660592b844..40e620825f 100644 --- a/tests/ParallelTypeCheckingTests/Tests/ComponentTests.args.txt +++ b/tests/ParallelTypeCheckingTests/Tests/ComponentTests.args.txt @@ -28,7 +28,7 @@ --doc:..\..\artifacts\obj\FSharp.Compiler.ComponentTests\Debug\net7.0\FSharp.Compiler.ComponentTests.xml --keyfile:$PACKAGES$\microsoft.dotnet.arcade.sdk\8.0.0-beta.22552.1\tools\snk/MSFT.snk --publicsign+ ---optimize- +--optimize+ --tailcalls- -r:$PACKAGES$\fluentassertions\5.10.3\lib\netcoreapp2.1\FluentAssertions.dll -r:..\..\artifacts\bin\FSharp.Build\Debug\netstandard2.0\FSharp.Build.dll diff --git a/tests/ParallelTypeCheckingTests/Tests/FCS.args.txt b/tests/ParallelTypeCheckingTests/Tests/FCS.args.txt index 5c056374fb..012de0b2cf 100644 --- a/tests/ParallelTypeCheckingTests/Tests/FCS.args.txt +++ b/tests/ParallelTypeCheckingTests/Tests/FCS.args.txt @@ -33,7 +33,7 @@ --doc:..\..\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSharp.Compiler.Service.xml --keyfile:$PACKAGES$\microsoft.dotnet.arcade.sdk\8.0.0-beta.22552.1\tools\snk/MSFT.snk --publicsign+ ---optimize- +--optimize+ --resource:..\..\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSComp.resources --resource:..\..\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\FSIstrings.resources --resource:..\..\artifacts\obj\FSharp.Compiler.Service\Debug\netstandard2.0\UtilsStrings.resources From 351b967f456431bfbde0bfcc0cf6f96bb8fbba8d Mon Sep 17 00:00:00 2001 From: janusz Date: Sun, 20 Nov 2022 02:09:48 +0000 Subject: [PATCH 15/21] Fix bugs - now works with FCS --- src/Compiler/Driver/OptimizeInputs.fs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index 19ce7f2402..237e35f172 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -163,7 +163,6 @@ let getPhase1Res (p: FileResults) = let getPhase2Res (p: FileResults) = p.Phase2 |> Option.get - |> fun (env, _) -> env let getPhase3Res (p: FileResults) = p.Phase3 @@ -212,15 +211,15 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu let getPhase1Res (p: FileResults) = p.Phase1 |> Option.get - |> fun ((env, _, _, hidden), _) -> env, hidden + |> fun ((env, file, _, hidden), _) -> env, file, hidden match phase with | Phase.Phase1 -> // take env from previous file - let env, hidingInfo = + let env, _, hidingInfo = previous |> Option.map getPhase1Res - |> Option.defaultValue (env0, hidingInfo0) + |> Option.defaultValue (env0, file, hidingInfo0) let inputs = env, hidingInfo, file let phase1Res = phase1 inputs res.Phase1 <- Some phase1Res @@ -235,14 +234,13 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu | Phase.Phase2 -> // take env from previous file - let env = + let env, _ = previous |> Option.map getPhase2Res - |> Option.defaultValue env0 - let hidingInfo = + |> Option.defaultValue (env0, file) + let _optEnv, file, hidingInfo = res |> getPhase1Res - |> snd let inputs = env, hidingInfo, file let phase2Res = phase2 inputs res.Phase2 <- Some phase2Res @@ -262,11 +260,13 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu previous |> Option.map getPhase3Res |> Option.defaultValue env0 - let hidingInfo = - previous - |> Option.map getPhase1Res - |> Option.map snd - |> Option.defaultValue hidingInfo0 + // impl file + let _, file = + res + |> getPhase2Res + let _phase1Env, _, hidingInfo = + res + |> getPhase1Res let inputs = env, hidingInfo, file let phase3Res = phase3 inputs res.Phase3 <- Some phase3Res From e7eae4fe4c286f9d86521808beb9b0d5413256bc Mon Sep 17 00:00:00 2001 From: janusz Date: Mon, 21 Nov 2022 09:38:25 +0000 Subject: [PATCH 16/21] changes --- src/Compiler/Optimize/Optimizer.fs | 32 ++- .../Code/GraphBasedOpt.fs | 207 ++++++++++++++++++ .../Code/GraphProcessing.fs | 95 ++++++++ .../Code/ParallelTypeChecking.fs | 2 + .../ParallelTypeCheckingTests.fsproj | 1 + 5 files changed, 327 insertions(+), 10 deletions(-) create mode 100644 tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs diff --git a/src/Compiler/Optimize/Optimizer.fs b/src/Compiler/Optimize/Optimizer.fs index 54665570cc..5037c32318 100644 --- a/src/Compiler/Optimize/Optimizer.fs +++ b/src/Compiler/Optimize/Optimizer.fs @@ -562,16 +562,16 @@ let BindInternalLocalVal cenv (v: Val) vval env = cenv.localInternalVals[v.Stamp] <- vval env -let BindExternalLocalVal cenv (v: Val) vval env = +let BindExternalLocalVal cenv (v: Val) vval env acc = let g = cenv.g let vval = if v.IsMutable then {vval with ValExprInfo=UnknownValue } else vval - let env = + let acc = match vval.ValExprInfo with - | UnknownValue -> env + | UnknownValue -> acc | _ -> - { env with localExternalVals=env.localExternalVals.Add (v.Stamp, vval) } + { acc with localExternalVals=acc.localExternalVals.Add (v.Stamp, vval) } // If we're compiling fslib then also bind the value as a non-local path to // allow us to resolve the compiler-non-local-references that arise from env.fs // @@ -579,14 +579,16 @@ let BindExternalLocalVal cenv (v: Val) vval env = // v, dereferencing it to find the corresponding signature Val, and adding an entry for the signature val. // // A similar code path exists in ilxgen.fs for the tables of "representations" for values - let env = - if g.compilingFSharpCore then + let env = + // TODO We ignore the fact this isn't delta-based for now. + if g.compilingFSharpCore then + failwith "FSharpCore optimization not supported on this PoC branch" // Passing an empty remap is sufficient for FSharp.Core.dll because it turns out the remapped type signature can // still be resolved. match tryRescopeVal g.fslibCcu Remap.Empty v with | ValueSome vref -> BindValueForFslib vref.nlr v vval env | _ -> env - else env + else acc env let rec BindValsInModuleOrNamespace cenv (mval: LazyModuleInfo) env = @@ -4272,7 +4274,16 @@ and OptimizeModuleDefs cenv (env, bindInfosColl) defs = let defs, (env, bindInfosColl) = List.mapFold (OptimizeModuleContents cenv) (env, bindInfosColl) defs let defs, minfos = List.unzip defs (defs, UnionOptimizationInfos minfos), (env, bindInfosColl) - + +and mergeSignatureHidingInfos (a: SignatureHidingInfo) (b: SignatureHidingInfo) = + { + SignatureHidingInfo.HiddenTycons = Zset.union a.HiddenTycons b.HiddenTycons + HiddenTyconReprs = Zset.union a.HiddenTyconReprs b.HiddenTyconReprs + HiddenVals = Zset.union a.HiddenVals b.HiddenVals + HiddenRecdFields = Zset.union a.HiddenRecdFields b.HiddenRecdFields + HiddenUnionCases = Zset.union a.HiddenUnionCases b.HiddenUnionCases + } + and OptimizeImplFileInternal cenv env isIncrementalFragment fsiMultiAssemblyEmit hidden implFile = let g = cenv.g let (CheckedImplFile (qname, pragmas, signature, contents, hasExplicitEntryPoint, isScript, anonRecdTypes, namedDebugPointsForInlinedCode)) = implFile @@ -4298,7 +4309,8 @@ and OptimizeImplFileInternal cenv env isIncrementalFragment fsiMultiAssemblyEmit else // This optimizes and builds minfo w.r.t. the signature let mexprR, minfo = OptimizeModuleExprWithSig cenv env signature contents - let hidden = ComputeSignatureHidingInfoAtAssemblyBoundary signature hidden + let hiddenDelta = ComputeSignatureHidingInfoAtAssemblyBoundary signature SignatureHidingInfo.Empty + let hidden = mergeSignatureHidingInfos hidden hiddenDelta let minfoExternal = AbstractLazyModulInfoByHiding true hidden minfo let env = // In F# interactive multi-assembly mode, internals are not accessible in the 'env' used intra-assembly @@ -4311,7 +4323,7 @@ and OptimizeImplFileInternal cenv env isIncrementalFragment fsiMultiAssemblyEmit let implFileR = CheckedImplFile (qname, pragmas, signature, contentsR, hasExplicitEntryPoint, isScript, anonRecdTypes, namedDebugPointsForInlinedCode) - env, implFileR, minfo, hidden + env , implFileR, minfo, hidden /// Entry point let OptimizeImplFile (settings, ccu, tcGlobals, tcVal, importMap, optEnv, isIncrementalFragment, fsiMultiAssemblyEmit, emitTailcalls, hidden, mimpls) = diff --git a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs new file mode 100644 index 0000000000..f86eef20fa --- /dev/null +++ b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs @@ -0,0 +1,207 @@ +module internal ParallelTypeCheckingTests.Code.GraphBasedOpt + +#nowarn "1182" + +open System +open System.Collections.Concurrent +open System.Collections.Generic +open System.IO +open System.Threading +open FSharp.Compiler +open FSharp.Compiler.CheckBasics +open FSharp.Compiler.CheckDeclarations +open FSharp.Compiler.CompilerConfig +open FSharp.Compiler.CompilerImports +open FSharp.Compiler.DiagnosticsLogger +open FSharp.Compiler.NameResolution +open FSharp.Compiler.OptimizeInputs +open FSharp.Compiler.Optimizer +open FSharp.Compiler.ParseAndCheckInputs +open FSharp.Compiler.TypedTreeOps +open ParallelTypeCheckingTests.FileInfoGathering +open ParallelTypeCheckingTests.Types +open ParallelTypeCheckingTests.Utils +open ParallelTypeCheckingTests +open ParallelTypeCheckingTests.DepResolving +open FSharp.Compiler.Syntax +open FSharp.Compiler.TcGlobals +open FSharp.Compiler.Text +open FSharp.Compiler.TypedTree +open Internal.Utilities.Library +open Internal.Utilities.Library.Extras + + +type OptimizeDuringCodeGen = bool -> Expr -> Expr +type OptimizeRes = + (IncrementalOptimizationEnv * CheckedImplFile * ImplFileOptimizationInfo * SignatureHidingInfo) * OptimizeDuringCodeGen + +type Optimize = + OptimizationSettings * + CcuThunk * + TcGlobals * + ConstraintSolver.TcValF * + Import.ImportMap * + IncrementalOptimizationEnv * + bool * + bool * + bool * + SignatureHidingInfo * + CheckedImplFile -> + OptimizeRes + +type PhaseInputs = IncrementalOptimizationEnv * SignatureHidingInfo * CheckedImplFile + +type Phase1Inputs = PhaseInputs +type Phase1Res = OptimizeRes +type Phase1Fun = Phase1Inputs -> Phase1Res + +type Phase2Inputs = PhaseInputs +type Phase2Res = IncrementalOptimizationEnv * CheckedImplFile +type Phase2Fun = Phase2Inputs -> Phase2Res + +type Phase3Inputs = PhaseInputs +type Phase3Res = IncrementalOptimizationEnv * CheckedImplFile +type Phase3Fun = Phase3Inputs -> Phase3Res + +type PhaseRes = + | Phase1 of Phase1Res + | Phase2 of Phase2Res + | Phase3 of Phase3Res + +type FileResultsComplete = + { + Phase1: Phase1Res + Phase2: Phase2Res + Phase3: Phase3Res + } +type CollectorInputs = FileResultsComplete[] +type CollectorOutputs = (CheckedImplFileAfterOptimization * ImplFileOptimizationInfo)[] * IncrementalOptimizationEnv + +let collectResults (inputs: CollectorInputs) : CollectorOutputs = + let files = + inputs + |> Array.map (fun {Phase1 = phase1; Phase2 = _phase2; Phase3 = phase3} -> + let (_, _, implFileOptData, _), optimizeDuringCodeGen = phase1 + let _, implFile = phase3 + let implFile = + { + ImplFile = implFile + OptimizeDuringCodeGen = optimizeDuringCodeGen + } + implFile, implFileOptData + ) + + let lastFilePhase1Env = + inputs + |> Array.last + |> fun {Phase1 = phase1} -> + let (optEnvPhase1, _, _, _), _ = phase1 + optEnvPhase1 + + files, lastFilePhase1Env + +type Phase = + | Phase1 + | Phase2 + | Phase3 +module Phase = + let all = [|Phase1; Phase2; Phase3|] + let prev (phase: Phase) = + match phase with + | Phase1 -> None + | Phase2 -> Some Phase1 + | Phase3 -> Some Phase2 + let next (phase: Phase) = + match phase with + | Phase1 -> Some Phase2 + | Phase2 -> Some Phase3 + | Phase3 -> None + +type FilePhaseFuncs = Phase1Fun * Phase2Fun * Phase3Fun +type FileResults = + { + mutable Phase1: Phase1Res option + mutable Phase2: Phase2Res option + mutable Phase3: Phase3Res option + } + with + member this.HasResult (phase: Phase) = + match phase with + | Phase.Phase1 -> this.Phase1 |> Option.isSome + | Phase.Phase2 -> this.Phase2 |> Option.isSome + | Phase.Phase3 -> this.Phase3 |> Option.isSome + static member Empty = + { + Phase1 = None + Phase2 = None + Phase3 = None + } + +type WorkItem = + | Phase1 of Phase1Inputs + | Phase2 of Phase2Inputs + | Phase3 of Phase3Inputs + +type Node = + { + Phase: Phase + Idx: FileIdx + } + with override this.ToString() = $"[{this.Idx}-{this.Phase}]" +module Node = + let make phase idx = { Idx = idx; Phase = phase } + + +let getPhase1Res (p: FileResults) = + p.Phase1 + |> Option.get + |> fun ((env, _, _, hidden), _) -> env, hidden + +let getPhase2Res (p: FileResults) = + p.Phase2 + |> Option.get + +let getPhase3Res (p: FileResults) = + p.Phase3 + |> Option.get + |> fun (env, _) -> env + +type IdxGraph = Graph + +type _Result = OptimizeRes + +let processNode ({Idx = idx; Phase = phase} as node : Node) (res: FileResults) : OptimizeRes = + failwith "" + +let goGraph (graph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFuncs) (files: CheckedImplFile[]) : CollectorOutputs = + // Create a 3x graph by cloning each file with its deps for each phase. Add links from phase3 -> phase2 -> phase1 + let graph = + graph + |> Seq.collect (fun (KeyValue(file, deps)) -> + // Create a node per each phase + Phase.all + |> Array.map (fun phase -> + let cur = Node.make phase file + let deps = + deps + |> Array.map (Node.make phase) + let prevNode = + Phase.prev phase + |> Option.map (fun prev -> Node.make prev file) + let deps = + match prevNode with + | Some prev -> Array.append deps [|prev|] + | None -> deps + cur, deps + ) + ) + |> readOnlyDict + + let results = + GraphProcessing.processGraphSimple + graph + (failwith "") + 1 + + + failwith "" diff --git a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs index 387371ae3a..fa10c3c20a 100644 --- a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs +++ b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs @@ -274,3 +274,98 @@ let processGraph<'Item, 'State, 'Result, 'FinalFileResult when 'Item: equality a ([||], emptyState) finals, state + +type Node2<'Item, 'Result> = + { + Info: NodeInfo<'Item> + mutable ProcessedDepsCount: int + mutable Result: 'Result option + } + +// TODO Could be replaced with a simpler recursive approach with memoised per-item results +let processGraphSimple<'Item, 'Result when 'Item: equality and 'Item: comparison> + (graph: Graph<'Item>) + // Accepts item and a list of item results. Handles combining results. + (doWork: 'Item -> ResultWrapper<'Item, 'Result>[] -> 'Result) + (parallelism: int) + : ResultWrapper<'Item, 'Result>[] = + let transitiveDeps = graph |> Graph.transitiveOpt + let dependants = graph |> Graph.reverse + + let makeNode (item: 'Item) : Node2<'Item, ResultWrapper<'Item, 'Result>> = + let info = + let exists = graph.ContainsKey item + if + not exists + || not (transitiveDeps.ContainsKey item) + || not (dependants.ContainsKey item) + then + failwith $"WHAT {item}" + + { + Item = item + Deps = graph[item] + TransitiveDeps = transitiveDeps[item] + Dependants = dependants[item] + } + + { + Info = info + Result = None + ProcessedDepsCount = 0 + } + + let nodes = graph.Keys |> Seq.map (fun item -> item, makeNode item) |> readOnlyDict + let lookup item = nodes[item] + let lookupMany items = items |> Array.map lookup + + let leaves = + nodes.Values + |> Seq.filter (fun n -> n.Info.Deps.Length = 0) + |> Seq.toArray + + printfn $"Node count: {nodes.Count}" + + let work + (node: Node2<'Item, ResultWrapper<'Item, 'Result>>) + : Node2<'Item, ResultWrapper<'Item, 'Result>>[] = + let _deps = lookupMany node.Info.Deps + let transitiveDeps = lookupMany node.Info.TransitiveDeps + let inputs = + transitiveDeps + |> Array.map (fun n -> n.Result |> Option.get) + let singleRes = doWork node.Info.Item inputs + let singleRes = + { + Item = node.Info.Item + Result = singleRes + } + node.Result <- Some singleRes + // Need to double-check that only one dependency schedules this dependant + let unblocked = + node.Info.Dependants + |> lookupMany + |> Array.filter (fun x -> + let pdc = + // TODO Not ideal, better ways most likely exist + lock x (fun () -> + x.ProcessedDepsCount <- x.ProcessedDepsCount + 1 + x.ProcessedDepsCount) + pdc = x.Info.Deps.Length + ) + unblocked + + use cts = new CancellationTokenSource() + + Parallel.processInParallel + leaves + work + parallelism + (fun processedCount -> processedCount = nodes.Count) + cts.Token + (fun x -> x.Info.Item.ToString()) + + let nodesArray = nodes.Values |> Seq.toArray + + nodesArray + |> Array.map (fun n -> n.Result.Value) diff --git a/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs b/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs index b4a037d84e..9e85287172 100644 --- a/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs +++ b/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs @@ -14,6 +14,8 @@ open FSharp.Compiler.CompilerConfig open FSharp.Compiler.CompilerImports open FSharp.Compiler.DiagnosticsLogger open FSharp.Compiler.NameResolution +open FSharp.Compiler.OptimizeInputs +open FSharp.Compiler.Optimizer open FSharp.Compiler.ParseAndCheckInputs open ParallelTypeCheckingTests.FileInfoGathering open ParallelTypeCheckingTests.Types diff --git a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj index 19069edd21..cb0f1defe1 100644 --- a/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj +++ b/tests/ParallelTypeCheckingTests/ParallelTypeCheckingTests.fsproj @@ -43,6 +43,7 @@ + From d253494f329b4ef89888808926267c92f69c0920 Mon Sep 17 00:00:00 2001 From: janusz Date: Tue, 22 Nov 2022 00:41:29 +0000 Subject: [PATCH 17/21] changes --- src/Compiler/Optimize/Optimizer.fs | 55 +++++++- src/Compiler/Optimize/Optimizer.fsi | 2 + src/Compiler/TypedTree/TypedTreeOps.fs | 10 +- src/Compiler/TypedTree/TypedTreeOps.fsi | 4 + .../Code/GraphBasedOpt.fs | 119 ++++++++++++++---- .../Code/GraphProcessing.fs | 89 ++++++++++++- 6 files changed, 248 insertions(+), 31 deletions(-) diff --git a/src/Compiler/Optimize/Optimizer.fs b/src/Compiler/Optimize/Optimizer.fs index 5037c32318..3cda7349b2 100644 --- a/src/Compiler/Optimize/Optimizer.fs +++ b/src/Compiler/Optimize/Optimizer.fs @@ -441,6 +441,7 @@ type MethodEnv = override x.ToString() = "" + type IncrementalOptimizationEnv = { /// An identifier to help with name generation latestBoundId: Ident option @@ -481,6 +482,54 @@ type IncrementalOptimizationEnv = override x.ToString() = "" +let empty = + { latestBoundId = None // Not used across files + dontInline = Zset.empty Int64.order // sum + typarInfos = [] // sum + functionVal = None + dontSplitVars = ValMap.Empty // sum + disableMethodSplitting = false // ? + localExternalVals = LayeredMap.Empty // sum + globalModuleInfos = LayeredMap.Empty // sum + methEnv = { pipelineCount = 0 } } // Not used across files + +let mergeMaps<'Key, 'Value when 'Key : comparison> (maps: Map<'Key, 'Value>[]) = + maps + |> Array.collect Map.toArray + |> Map.ofArray + +let mergeEnvs (envs: IncrementalOptimizationEnv[]): IncrementalOptimizationEnv = + // TODO use a single HashSet for perf? + let dontInline = + envs + |> Array.collect (fun e -> e.dontInline |> Zset.elements |> List.toArray) + |> fun xs -> Zset.Create (Int64.order, xs) + let typarInfos = envs |> Array.toList |> List.collect (fun e -> e.typarInfos) + let dontSplitVars = + envs + |> Seq.collect (fun e -> e.dontSplitVars.Contents |> Map.toSeq) + |> Seq.toArray + |> ValMap.OfArray' + let localExternalVals = + envs + |> Array.map (fun e -> e.localExternalVals) + |> mergeMaps + let globalModuleInfos = + envs + |> Array.map (fun e -> e.globalModuleInfos) + |> mergeMaps + { + latestBoundId = None // Not used across files + dontInline = dontInline// sum + typarInfos = typarInfos // sum + functionVal = None // Not used across files + dontSplitVars = dontSplitVars // sum + disableMethodSplitting = false // not used across files + localExternalVals = localExternalVals // sum + globalModuleInfos = globalModuleInfos // sum + methEnv = { pipelineCount = 0 } // Not used across files + } + //------------------------------------------------------------------------- // IsPartialExprVal - is the expr fully known? //------------------------------------------------------------------------- @@ -562,16 +611,16 @@ let BindInternalLocalVal cenv (v: Val) vval env = cenv.localInternalVals[v.Stamp] <- vval env -let BindExternalLocalVal cenv (v: Val) vval env acc = +let BindExternalLocalVal cenv (v: Val) vval env = let g = cenv.g let vval = if v.IsMutable then {vval with ValExprInfo=UnknownValue } else vval let acc = match vval.ValExprInfo with - | UnknownValue -> acc + | UnknownValue -> env | _ -> - { acc with localExternalVals=acc.localExternalVals.Add (v.Stamp, vval) } + { env with localExternalVals=env.localExternalVals.Add (v.Stamp, vval) } // If we're compiling fslib then also bind the value as a non-local path to // allow us to resolve the compiler-non-local-references that arise from env.fs // diff --git a/src/Compiler/Optimize/Optimizer.fsi b/src/Compiler/Optimize/Optimizer.fsi index 76b7881f48..5ced7ef65b 100644 --- a/src/Compiler/Optimize/Optimizer.fsi +++ b/src/Compiler/Optimize/Optimizer.fsi @@ -61,6 +61,8 @@ type CcuOptimizationInfo = LazyModuleInfo type IncrementalOptimizationEnv = static member Empty: IncrementalOptimizationEnv +val mergeEnvs : IncrementalOptimizationEnv[] -> IncrementalOptimizationEnv + /// For building optimization environments incrementally val internal BindCcu: CcuThunk -> CcuOptimizationInfo -> IncrementalOptimizationEnv -> TcGlobals -> IncrementalOptimizationEnv diff --git a/src/Compiler/TypedTree/TypedTreeOps.fs b/src/Compiler/TypedTree/TypedTreeOps.fs index f57954f7bf..a000c3e47a 100644 --- a/src/Compiler/TypedTree/TypedTreeOps.fs +++ b/src/Compiler/TypedTree/TypedTreeOps.fs @@ -86,7 +86,8 @@ type ValMap<'T>(imap: StampMap<'T>) = member _.Remove (v: Val) = ValMap (imap.Remove(v.Stamp)) static member Empty = ValMap<'T> Map.empty member _.IsEmpty = imap.IsEmpty - static member OfList vs = (vs, ValMap<'T>.Empty) ||> List.foldBack (fun (x, y) acc -> acc.Add x y) + static member OfList vs = (vs, ValMap<'T>.Empty) ||> List.foldBack (fun (x, y) acc -> acc.Add x y) + static member OfArray' (vs: (Stamp * 'T)[]) = (vs, ValMap<'T>.Empty) ||> Array.foldBack (fun (x, y) acc -> ValMap (acc.Contents.Add(x, y))) //-------------------------------------------------------------------------- // renamings @@ -4641,6 +4642,13 @@ type SignatureHidingInfo = HiddenVals = Zset.empty valOrder HiddenRecdFields = Zset.empty recdFieldRefOrder HiddenUnionCases = Zset.empty unionCaseRefOrder } + + static member Union (a: SignatureHidingInfo) (b: SignatureHidingInfo): SignatureHidingInfo = + { HiddenTycons = Zset.union a.HiddenTycons b.HiddenTycons + HiddenTyconReprs = Zset.union a.HiddenTyconReprs b.HiddenTyconReprs + HiddenVals = Zset.union a.HiddenVals b.HiddenVals + HiddenRecdFields = Zset.union a.HiddenRecdFields b.HiddenRecdFields + HiddenUnionCases = Zset.union a.HiddenUnionCases b.HiddenUnionCases } let addValRemap v vNew tmenv = { tmenv with valRemap= tmenv.valRemap.Add v (mkLocalValRef vNew) } diff --git a/src/Compiler/TypedTree/TypedTreeOps.fsi b/src/Compiler/TypedTree/TypedTreeOps.fsi index 00d838e04d..fa6ae8d2b2 100755 --- a/src/Compiler/TypedTree/TypedTreeOps.fsi +++ b/src/Compiler/TypedTree/TypedTreeOps.fsi @@ -424,6 +424,8 @@ type ValMap<'T> = static member Empty: ValMap<'T> static member OfList: (Val * 'T) list -> ValMap<'T> + + static member OfArray': (Stamp * 'T)[] -> ValMap<'T> /// Mutable data structure mapping Val's to T based on stamp keys [] @@ -1269,6 +1271,8 @@ type SignatureHidingInfo = /// The empty table representing no hiding static member Empty: SignatureHidingInfo + + static member Union: SignatureHidingInfo -> SignatureHidingInfo -> SignatureHidingInfo /// Compute the remapping information implied by a signature being inferred for a particular implementation val ComputeRemappingFromImplementationToSignature: diff --git a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs index f86eef20fa..88986dc903 100644 --- a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs +++ b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs @@ -63,10 +63,43 @@ type Phase3Inputs = PhaseInputs type Phase3Res = IncrementalOptimizationEnv * CheckedImplFile type Phase3Fun = Phase3Inputs -> Phase3Res +type Phase = + | Phase1 + | Phase2 + | Phase3 +module Phase = + let all = [|Phase1; Phase2; Phase3|] + let prev (phase: Phase) = + match phase with + | Phase1 -> None + | Phase2 -> Some Phase1 + | Phase3 -> Some Phase2 + let next (phase: Phase) = + match phase with + | Phase1 -> Some Phase2 + | Phase2 -> Some Phase3 + | Phase3 -> None + type PhaseRes = | Phase1 of Phase1Res | Phase2 of Phase2Res | Phase3 of Phase3Res + with + member x.Which = + match x with + | Phase1 _ -> Phase.Phase1 + | Phase2 _ -> Phase.Phase2 + | Phase3 _ -> Phase.Phase3 + member x.Get1() = + match x with + | Phase1 x -> x + | Phase2 _ + | Phase3 _ -> failwith $"Called {nameof(x.Get1)} but this is {x.Which}" + member x.Get2() = + match x with + | Phase2 x -> x + | Phase1 _ + | Phase3 _ -> failwith $"Called {nameof(x.Get2)} but this is {x.Which}" type FileResultsComplete = { @@ -100,23 +133,6 @@ let collectResults (inputs: CollectorInputs) : CollectorOutputs = files, lastFilePhase1Env -type Phase = - | Phase1 - | Phase2 - | Phase3 -module Phase = - let all = [|Phase1; Phase2; Phase3|] - let prev (phase: Phase) = - match phase with - | Phase1 -> None - | Phase2 -> Some Phase1 - | Phase3 -> Some Phase2 - let next (phase: Phase) = - match phase with - | Phase1 -> Some Phase2 - | Phase2 -> Some Phase3 - | Phase3 -> None - type FilePhaseFuncs = Phase1Fun * Phase2Fun * Phase3Fun type FileResults = { @@ -130,6 +146,10 @@ type FileResults = | Phase.Phase1 -> this.Phase1 |> Option.isSome | Phase.Phase2 -> this.Phase2 |> Option.isSome | Phase.Phase3 -> this.Phase3 |> Option.isSome + member x.Get1 () = x.Phase1 |> Option.get + member x.Get2 () = x.Phase2 |> Option.get + member x.Get3 () = x.Phase3 |> Option.get + static member Empty = { Phase1 = None @@ -137,6 +157,13 @@ type FileResults = Phase3 = None } +module FileResults = + let complete (results: FileResults) = + let {FileResults.Phase1 = phase1; Phase2 = phase2; Phase3 = phase3} = results + match phase1, phase2, phase3 with + | Some phase1, Some phase2, Some phase3 -> {FileResultsComplete.Phase1 = phase1; Phase2 = phase2; Phase3 = phase3} + | _ -> failwith $"Unexpected lack of results" + type WorkItem = | Phase1 of Phase1Inputs | Phase2 of Phase2Inputs @@ -173,10 +200,33 @@ type _Result = OptimizeRes let processNode ({Idx = idx; Phase = phase} as node : Node) (res: FileResults) : OptimizeRes = failwith "" -let goGraph (graph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFuncs) (files: CheckedImplFile[]) : CollectorOutputs = +let collectResults (inputs: CollectorInputs) : CollectorOutputs = + let files = + inputs + |> Array.map (fun {Phase1 = phase1; Phase2 = _phase2; Phase3 = phase3} -> + let (_, _, implFileOptData, _), optimizeDuringCodeGen = phase1 + let _, implFile = phase3 + let implFile = + { + ImplFile = implFile + OptimizeDuringCodeGen = optimizeDuringCodeGen + } + implFile, implFileOptData + ) + + let lastFilePhase1Env = + inputs + |> Array.last + |> fun {Phase1 = phase1} -> + let (optEnvPhase1, _, _, _), _ = phase1 + optEnvPhase1 + + files, lastFilePhase1Env + +let goGraph (idxGraph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFuncs) (files: CheckedImplFile[]) : CollectorOutputs = // Create a 3x graph by cloning each file with its deps for each phase. Add links from phase3 -> phase2 -> phase1 let graph = - graph + idxGraph |> Seq.collect (fun (KeyValue(file, deps)) -> // Create a node per each phase Phase.all @@ -196,12 +246,33 @@ let goGraph (graph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, phase ) ) |> readOnlyDict + + let transitiveGraph = + graph + |> Graph.transitiveOpt let results = - GraphProcessing.processGraphSimple - graph - (failwith "") - 1 + Array.init files.Length (fun _ -> FileResults.Empty) + let getRes (FileIdx idx) = results[idx] + + let work (x: Node) : unit = + let {Idx=idx; Phase=phase} = x + let res = getRes idx + let deps = transitiveGraph[x] + + match phase with + | Phase.Phase1 -> + failwith "" + + | _ -> failwith "" + + failwith "" + GraphProcessing.processGraphSimpler + graph + (failwith "") + 1 - failwith "" + let completeResults = results |> Array.map FileResults.complete + let collected = collectResults completeResults + collected diff --git a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs index fa10c3c20a..1a0382eeff 100644 --- a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs +++ b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs @@ -1,6 +1,7 @@ /// Parallel processing of graph of work items with dependencies module ParallelTypeCheckingTests.GraphProcessing +open System.Collections.Concurrent open System.Collections.Generic open System.Threading @@ -264,9 +265,6 @@ let processGraph<'Item, 'State, 'Result, 'FinalFileResult when 'Item: equality a nodesArray |> Array.filter (fun node -> includeInFinalState node.Info.Item) |> Array.sortBy (fun node -> node.Info.Item) - |> fun nodes -> - // printfn $"%+A{nodes |> Array.map (fun n -> n.Info.Item.ToString())}" - nodes |> Array.fold (fun (fileResults, state) node -> let fileResult, state = folder state (node.Result.Value |> snd) @@ -369,3 +367,88 @@ let processGraphSimple<'Item, 'Result when 'Item: equality and 'Item: comparison nodesArray |> Array.map (fun n -> n.Result.Value) + + +/// Used for processing +type NodeInfo3<'Item> = + { + Item: 'Item + Deps: 'Item[] + Dependants: 'Item[] + } + +type Node3<'Item> = + { + Info: NodeInfo3<'Item> + mutable ProcessedDepsCount: int + } + +/// Graph processing that doesn't handle results but just invokes the worker when dependencies are ready +let processGraphSimpler<'Item when 'Item: equality and 'Item: comparison> + (graph: Graph<'Item>) + // Accepts item and a list of item results. Handles combining results. + (doWork: 'Item -> unit) + (parallelism: int) + : unit + = + let dependants = graph |> Graph.reverse + + let makeNode (item: 'Item) : Node3<'Item> = + let info = + let exists = graph.ContainsKey item + if + not exists + || not (dependants.ContainsKey item) + then + failwith $"WHAT {item}" + { + Item = item + Deps = graph[item] + Dependants = dependants[item] + } + + { + Info = info + ProcessedDepsCount = 0 + } + + let nodes = graph.Keys |> Seq.map (fun item -> item, makeNode item) |> readOnlyDict + let lookup item = nodes[item] + let lookupMany items = items |> Array.map lookup + + let leaves = + nodes.Values + |> Seq.filter (fun n -> n.Info.Deps.Length = 0) + |> Seq.toArray + + printfn $"Node count: {nodes.Count}" + + let work + (node: Node3<'Item>) + : Node3<'Item>[] + = + let _deps = lookupMany node.Info.Deps + doWork node.Info.Item + // Need to double-check that only one dependency schedules this dependant + let unblocked = + node.Info.Dependants + |> lookupMany + |> Array.filter (fun x -> + let pdc = + // TODO Not ideal, better ways most likely exist + lock x (fun () -> + x.ProcessedDepsCount <- x.ProcessedDepsCount + 1 + x.ProcessedDepsCount) + pdc = x.Info.Deps.Length + ) + unblocked + + use cts = new CancellationTokenSource() + + Parallel.processInParallel + leaves + work + parallelism + (fun processedCount -> processedCount = nodes.Count) + cts.Token + (fun x -> x.Info.Item.ToString()) From 4003ac3f5cdd3cf30359db32c48c67ea5c2f0316 Mon Sep 17 00:00:00 2001 From: janusz Date: Tue, 22 Nov 2022 19:29:33 +0000 Subject: [PATCH 18/21] changes --- src/Compiler/Driver/OptimizeInputs.fs | 67 ++---- src/Compiler/Driver/OptimizeInputs.fsi | 13 +- src/Compiler/Driver/OptimizeTypes.fs | 87 ++++++++ src/Compiler/FSharp.Compiler.Service.fsproj | 1 + src/Compiler/Optimize/Optimizer.fs | 35 ++- src/Compiler/Optimize/Optimizer.fsi | 2 +- .../Code/GraphBasedOpt.fs | 203 ++++++------------ .../Tests/TestCompilation.fs | 42 ++++ 8 files changed, 247 insertions(+), 203 deletions(-) create mode 100644 src/Compiler/Driver/OptimizeTypes.fs diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index 237e35f172..c2fd81a1c7 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -7,6 +7,7 @@ open System.Diagnostics open System.IO open System.Threading open FSharp.Compiler.Optimizer +open FSharp.Compiler.Service.Driver.OptimizeTypes open Internal.Utilities.Library open FSharp.Compiler open FSharp.Compiler.AbstractIL.IL @@ -52,47 +53,6 @@ let GetInitialOptimizationEnv (tcImports: TcImports, tcGlobals: TcGlobals) = let optEnv = List.fold (AddExternalCcuToOptimizationEnv tcGlobals) optEnv ccuinfos optEnv -type OptimizeDuringCodeGen = bool -> Expr -> Expr -type OptimizeRes = - (IncrementalOptimizationEnv * CheckedImplFile * ImplFileOptimizationInfo * SignatureHidingInfo) * OptimizeDuringCodeGen - -type Optimize = - OptimizationSettings * - CcuThunk * - TcGlobals * - ConstraintSolver.TcValF * - Import.ImportMap * - IncrementalOptimizationEnv * - bool * - bool * - bool * - SignatureHidingInfo * - CheckedImplFile -> - OptimizeRes - -type PhaseInputs = IncrementalOptimizationEnv * SignatureHidingInfo * CheckedImplFile - -type Phase1Inputs = PhaseInputs -type Phase1Res = OptimizeRes -type Phase1Fun = Phase1Inputs -> Phase1Res - -type Phase2Inputs = PhaseInputs -type Phase2Res = IncrementalOptimizationEnv * CheckedImplFile -type Phase2Fun = Phase2Inputs -> Phase2Res - -type Phase3Inputs = PhaseInputs -type Phase3Res = IncrementalOptimizationEnv * CheckedImplFile -type Phase3Fun = Phase3Inputs -> Phase3Res - -type FileResultsComplete = - { - Phase1: Phase1Res - Phase2: Phase2Res - Phase3: Phase3Res - } -type CollectorInputs = FileResultsComplete[] -type CollectorOutputs = (CheckedImplFileAfterOptimization * ImplFileOptimizationInfo)[] * IncrementalOptimizationEnv - let collectResults (inputs: CollectorInputs) : CollectorOutputs = let files = inputs @@ -300,7 +260,17 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu let collected = results |> collectResults collected -let mutable UseParallelOptimizer: bool = false +type Goer = IReadOnlyDictionary -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs + +let mutable goer: Goer option = None + +[] +type OptimizerMode = + | Sequential + | PartiallyParallel + | GraphBased + +let mutable OptimizerMode: OptimizerMode = OptimizerMode.Sequential let ApplyAllOptimizations ( @@ -453,11 +423,20 @@ let ApplyAllOptimizations env, implFile let results, optEnvFirstLoop = - if UseParallelOptimizer then + match OptimizerMode with + | OptimizerMode.GraphBased -> + let graph = + [||] + |> readOnlyDict + let goer = goer.Value + let a, b = + goer graph env0 (phase1, phase2, phase3) (implFiles |> List.toArray) + a |> Array.toList, b + | OptimizerMode.PartiallyParallel -> let a, b = go env0 (phase1, phase2, phase3) (implFiles |> List.toArray) a |> Array.toList, b - else + | OptimizerMode.Sequential -> let results, (optEnvFirstLoop, _, _, _) = ((optEnv0, optEnv0, optEnv0, SignatureHidingInfo.Empty), implFiles) diff --git a/src/Compiler/Driver/OptimizeInputs.fsi b/src/Compiler/Driver/OptimizeInputs.fsi index 33881ae6cb..6552eb4e4e 100644 --- a/src/Compiler/Driver/OptimizeInputs.fsi +++ b/src/Compiler/Driver/OptimizeInputs.fsi @@ -12,6 +12,8 @@ open FSharp.Compiler.Import open FSharp.Compiler.Optimizer open FSharp.Compiler.TcGlobals open FSharp.Compiler.TypedTree +open System.Collections.Generic +open FSharp.Compiler.Service.Driver.OptimizeTypes val GetGeneratedILModuleName: CompilerTarget -> string -> string @@ -50,4 +52,13 @@ val NormalizeAssemblyRefs: CompilationThreadToken * ILGlobals * TcImports -> (IL val GetGeneratedILModuleName: CompilerTarget -> string -> string -val mutable UseParallelOptimizer: bool \ No newline at end of file +type FilePhaseFuncs = Phase1Fun * Phase2Fun * Phase3Fun +type Goer = IReadOnlyDictionary -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs +val mutable goer: Goer option + +type OptimizerMode = + | Sequential + | PartiallyParallel + | GraphBased + +val mutable OptimizerMode: OptimizerMode \ No newline at end of file diff --git a/src/Compiler/Driver/OptimizeTypes.fs b/src/Compiler/Driver/OptimizeTypes.fs new file mode 100644 index 0000000000..e2f15dd6ba --- /dev/null +++ b/src/Compiler/Driver/OptimizeTypes.fs @@ -0,0 +1,87 @@ +module internal FSharp.Compiler.Service.Driver.OptimizeTypes + +open FSharp.Compiler +open FSharp.Compiler.Optimizer +open FSharp.Compiler.TcGlobals +open FSharp.Compiler.TypedTree +open FSharp.Compiler.TypedTreeOps + + +type OptimizeDuringCodeGen = bool -> Expr -> Expr +type OptimizeRes = + (IncrementalOptimizationEnv * CheckedImplFile * ImplFileOptimizationInfo * SignatureHidingInfo) * OptimizeDuringCodeGen + +type Optimize = + OptimizationSettings * + CcuThunk * + TcGlobals * + ConstraintSolver.TcValF * + Import.ImportMap * + IncrementalOptimizationEnv * + bool * + bool * + bool * + SignatureHidingInfo * + CheckedImplFile -> + OptimizeRes + +type PhaseInputs = IncrementalOptimizationEnv * SignatureHidingInfo * CheckedImplFile + +type Phase1Inputs = PhaseInputs +type Phase1Res = OptimizeRes +type Phase1Fun = Phase1Inputs -> Phase1Res + +type Phase2Inputs = PhaseInputs +type Phase2Res = IncrementalOptimizationEnv * CheckedImplFile +type Phase2Fun = Phase2Inputs -> Phase2Res + +type Phase3Inputs = PhaseInputs +type Phase3Res = IncrementalOptimizationEnv * CheckedImplFile +type Phase3Fun = Phase3Inputs -> Phase3Res + +type Phase = + | Phase1 + | Phase2 + | Phase3 +module Phase = + let all = [|Phase1; Phase2; Phase3|] + let prev (phase: Phase) = + match phase with + | Phase1 -> None + | Phase2 -> Some Phase1 + | Phase3 -> Some Phase2 + let next (phase: Phase) = + match phase with + | Phase1 -> Some Phase2 + | Phase2 -> Some Phase3 + | Phase3 -> None + +type PhaseRes = + | Phase1 of Phase1Res + | Phase2 of Phase2Res + | Phase3 of Phase3Res + with + member x.Which = + match x with + | Phase1 _ -> Phase.Phase1 + | Phase2 _ -> Phase.Phase2 + | Phase3 _ -> Phase.Phase3 + member x.Get1() = + match x with + | Phase1 x -> x + | Phase2 _ + | Phase3 _ -> failwith $"Called {nameof(x.Get1)} but this is {x.Which}" + member x.Get2() = + match x with + | Phase2 x -> x + | Phase1 _ + | Phase3 _ -> failwith $"Called {nameof(x.Get2)} but this is {x.Which}" + +type FileResultsComplete = + { + Phase1: Phase1Res + Phase2: Phase2Res + Phase3: Phase3Res + } +type CollectorInputs = FileResultsComplete[] +type CollectorOutputs = (CheckedImplFileAfterOptimization * ImplFileOptimizationInfo)[] * IncrementalOptimizationEnv diff --git a/src/Compiler/FSharp.Compiler.Service.fsproj b/src/Compiler/FSharp.Compiler.Service.fsproj index 73624e0c27..2298d7be5e 100644 --- a/src/Compiler/FSharp.Compiler.Service.fsproj +++ b/src/Compiler/FSharp.Compiler.Service.fsproj @@ -388,6 +388,7 @@ + diff --git a/src/Compiler/Optimize/Optimizer.fs b/src/Compiler/Optimize/Optimizer.fs index 3cda7349b2..b2b1338d10 100644 --- a/src/Compiler/Optimize/Optimizer.fs +++ b/src/Compiler/Optimize/Optimizer.fs @@ -482,23 +482,15 @@ type IncrementalOptimizationEnv = override x.ToString() = "" -let empty = - { latestBoundId = None // Not used across files - dontInline = Zset.empty Int64.order // sum - typarInfos = [] // sum - functionVal = None - dontSplitVars = ValMap.Empty // sum - disableMethodSplitting = false // ? - localExternalVals = LayeredMap.Empty // sum - globalModuleInfos = LayeredMap.Empty // sum - methEnv = { pipelineCount = 0 } } // Not used across files - let mergeMaps<'Key, 'Value when 'Key : comparison> (maps: Map<'Key, 'Value>[]) = maps |> Array.collect Map.toArray |> Map.ofArray -let mergeEnvs (envs: IncrementalOptimizationEnv[]): IncrementalOptimizationEnv = + + +let mergeEnvs (env0: IncrementalOptimizationEnv) (envs: IncrementalOptimizationEnv[]): IncrementalOptimizationEnv = + let envs = Array.append [|env0|] envs // TODO use a single HashSet for perf? let dontInline = envs @@ -519,15 +511,16 @@ let mergeEnvs (envs: IncrementalOptimizationEnv[]): IncrementalOptimizationEnv = |> Array.map (fun e -> e.globalModuleInfos) |> mergeMaps { - latestBoundId = None // Not used across files - dontInline = dontInline// sum - typarInfos = typarInfos // sum - functionVal = None // Not used across files - dontSplitVars = dontSplitVars // sum - disableMethodSplitting = false // not used across files - localExternalVals = localExternalVals // sum - globalModuleInfos = globalModuleInfos // sum - methEnv = { pipelineCount = 0 } // Not used across files + env0 with + //latestBoundId = env0.latestBoundId // Not used across files + dontInline = dontInline// sum + typarInfos = typarInfos // sum + //functionVal = None // Not used across files + dontSplitVars = dontSplitVars // sum + //disableMethodSplitting = false // not used across files + localExternalVals = localExternalVals // sum + globalModuleInfos = globalModuleInfos // sum + //methEnv = { pipelineCount = 0 } // Not used across files } //------------------------------------------------------------------------- diff --git a/src/Compiler/Optimize/Optimizer.fsi b/src/Compiler/Optimize/Optimizer.fsi index 5ced7ef65b..95b65a5602 100644 --- a/src/Compiler/Optimize/Optimizer.fsi +++ b/src/Compiler/Optimize/Optimizer.fsi @@ -61,7 +61,7 @@ type CcuOptimizationInfo = LazyModuleInfo type IncrementalOptimizationEnv = static member Empty: IncrementalOptimizationEnv -val mergeEnvs : IncrementalOptimizationEnv[] -> IncrementalOptimizationEnv +val mergeEnvs : IncrementalOptimizationEnv -> IncrementalOptimizationEnv[] -> IncrementalOptimizationEnv /// For building optimization environments incrementally val internal BindCcu: diff --git a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs index 88986dc903..6898621db3 100644 --- a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs +++ b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs @@ -2,114 +2,16 @@ #nowarn "1182" -open System -open System.Collections.Concurrent -open System.Collections.Generic open System.IO -open System.Threading -open FSharp.Compiler -open FSharp.Compiler.CheckBasics -open FSharp.Compiler.CheckDeclarations -open FSharp.Compiler.CompilerConfig -open FSharp.Compiler.CompilerImports -open FSharp.Compiler.DiagnosticsLogger -open FSharp.Compiler.NameResolution -open FSharp.Compiler.OptimizeInputs open FSharp.Compiler.Optimizer -open FSharp.Compiler.ParseAndCheckInputs +open FSharp.Compiler.Service.Driver.OptimizeTypes open FSharp.Compiler.TypedTreeOps -open ParallelTypeCheckingTests.FileInfoGathering -open ParallelTypeCheckingTests.Types open ParallelTypeCheckingTests.Utils open ParallelTypeCheckingTests -open ParallelTypeCheckingTests.DepResolving -open FSharp.Compiler.Syntax -open FSharp.Compiler.TcGlobals -open FSharp.Compiler.Text open FSharp.Compiler.TypedTree open Internal.Utilities.Library open Internal.Utilities.Library.Extras - -type OptimizeDuringCodeGen = bool -> Expr -> Expr -type OptimizeRes = - (IncrementalOptimizationEnv * CheckedImplFile * ImplFileOptimizationInfo * SignatureHidingInfo) * OptimizeDuringCodeGen - -type Optimize = - OptimizationSettings * - CcuThunk * - TcGlobals * - ConstraintSolver.TcValF * - Import.ImportMap * - IncrementalOptimizationEnv * - bool * - bool * - bool * - SignatureHidingInfo * - CheckedImplFile -> - OptimizeRes - -type PhaseInputs = IncrementalOptimizationEnv * SignatureHidingInfo * CheckedImplFile - -type Phase1Inputs = PhaseInputs -type Phase1Res = OptimizeRes -type Phase1Fun = Phase1Inputs -> Phase1Res - -type Phase2Inputs = PhaseInputs -type Phase2Res = IncrementalOptimizationEnv * CheckedImplFile -type Phase2Fun = Phase2Inputs -> Phase2Res - -type Phase3Inputs = PhaseInputs -type Phase3Res = IncrementalOptimizationEnv * CheckedImplFile -type Phase3Fun = Phase3Inputs -> Phase3Res - -type Phase = - | Phase1 - | Phase2 - | Phase3 -module Phase = - let all = [|Phase1; Phase2; Phase3|] - let prev (phase: Phase) = - match phase with - | Phase1 -> None - | Phase2 -> Some Phase1 - | Phase3 -> Some Phase2 - let next (phase: Phase) = - match phase with - | Phase1 -> Some Phase2 - | Phase2 -> Some Phase3 - | Phase3 -> None - -type PhaseRes = - | Phase1 of Phase1Res - | Phase2 of Phase2Res - | Phase3 of Phase3Res - with - member x.Which = - match x with - | Phase1 _ -> Phase.Phase1 - | Phase2 _ -> Phase.Phase2 - | Phase3 _ -> Phase.Phase3 - member x.Get1() = - match x with - | Phase1 x -> x - | Phase2 _ - | Phase3 _ -> failwith $"Called {nameof(x.Get1)} but this is {x.Which}" - member x.Get2() = - match x with - | Phase2 x -> x - | Phase1 _ - | Phase3 _ -> failwith $"Called {nameof(x.Get2)} but this is {x.Which}" - -type FileResultsComplete = - { - Phase1: Phase1Res - Phase2: Phase2Res - Phase3: Phase3Res - } -type CollectorInputs = FileResultsComplete[] -type CollectorOutputs = (CheckedImplFileAfterOptimization * ImplFileOptimizationInfo)[] * IncrementalOptimizationEnv - let collectResults (inputs: CollectorInputs) : CollectorOutputs = let files = inputs @@ -178,11 +80,10 @@ type Node = module Node = let make phase idx = { Idx = idx; Phase = phase } - let getPhase1Res (p: FileResults) = p.Phase1 |> Option.get - |> fun ((env, _, _, hidden), _) -> env, hidden + |> fun ((env, file, _, hidden), _) -> env, file, hidden let getPhase2Res (p: FileResults) = p.Phase2 @@ -197,32 +98,12 @@ type IdxGraph = Graph type _Result = OptimizeRes -let processNode ({Idx = idx; Phase = phase} as node : Node) (res: FileResults) : OptimizeRes = - failwith "" - -let collectResults (inputs: CollectorInputs) : CollectorOutputs = - let files = - inputs - |> Array.map (fun {Phase1 = phase1; Phase2 = _phase2; Phase3 = phase3} -> - let (_, _, implFileOptData, _), optimizeDuringCodeGen = phase1 - let _, implFile = phase3 - let implFile = - { - ImplFile = implFile - OptimizeDuringCodeGen = optimizeDuringCodeGen - } - implFile, implFileOptData - ) - - let lastFilePhase1Env = - inputs - |> Array.last - |> fun {Phase1 = phase1} -> - let (optEnvPhase1, _, _, _), _ = phase1 - optEnvPhase1 - - files, lastFilePhase1Env - +let mergeHidingInfos (empty: SignatureHidingInfo) (infos: SignatureHidingInfo[]): SignatureHidingInfo = + infos + |> Array.fold SignatureHidingInfo.Union empty + +type Goer = IdxGraph -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs + let goGraph (idxGraph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFuncs) (files: CheckedImplFile[]) : CollectorOutputs = // Create a 3x graph by cloning each file with its deps for each phase. Add links from phase3 -> phase2 -> phase1 let graph = @@ -247,30 +128,80 @@ let goGraph (idxGraph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, ph ) |> readOnlyDict - let transitiveGraph = - graph + let mergeEnvs envs = + mergeEnvs env0 envs + + let transitiveIdxGraph = + idxGraph |> Graph.transitiveOpt let results = Array.init files.Length (fun _ -> FileResults.Empty) let getRes (FileIdx idx) = results[idx] + let hidingInfo0 = SignatureHidingInfo.Empty let work (x: Node) : unit = let {Idx=idx; Phase=phase} = x + let file = files[idx.Idx] let res = getRes idx - let deps = transitiveGraph[x] + let depResults = + transitiveIdxGraph[idx] + |> Array.map getRes match phase with | Phase.Phase1 -> - failwith "" - - | _ -> failwith "" - - failwith "" + // take env and hidingInfo from dependencies + let env = + depResults + |> Array.map (fun r -> + let (a,_b,_c,_d), _e = r.Get1() + a + ) + |> mergeEnvs + let hidingInfo = + depResults + |> Array.map (fun r -> + let (_a,_b,_c,d), _e = r.Get1() + d + ) + |> mergeHidingInfos hidingInfo0 + let inputs = env, hidingInfo, file + let phase1Res = phase1 inputs + res.Phase1 <- Some phase1Res + | Phase.Phase2 -> + // take env from dependencies + let env = + depResults + |> Array.map (fun r -> + let a,_b = r.Get2() + a + ) + |> mergeEnvs + // Get file and hidingInfo from phase1 of the current file + let (_optEnv, file, _, hidingInfo), _ = res.Get1() + let inputs = env, hidingInfo, file + let phase2Res = phase2 inputs + res.Phase2 <- Some phase2Res + | Phase.Phase3 -> + // take env from dependencies + let env = + depResults + |> Array.map (fun r -> + let a,_b = r.Get3() + a + ) + |> mergeEnvs + // Get file and hidingInfo from phase1 of the current file + let (_optEnv, _, _, hidingInfo), _ = res.Get1() + // Get impl file from phase2 + let _, file = res.Get2() + let inputs = env, hidingInfo, file + let phase3Res = phase3 inputs + res.Phase3 <- Some phase3Res GraphProcessing.processGraphSimpler graph - (failwith "") + work 1 let completeResults = results |> Array.map FileResults.complete diff --git a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs index 93f43a9ef6..36e2752265 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs @@ -206,6 +206,47 @@ let d (c: CType) = ] |> FProject.Make CompileOutput.Library + let fullyParallel = + [ + "A.fs", + """ +module A +let x = 1 +""" + "B.fs", + """ +module B +let x = 1 +""" + "C.fs", + """ +module C +let x = 1 +""" + "D.fs", + """ +module D +let x = 1 +""" + "E.fs", + """ +module E +let x = 1 +""" + "F.fs", + """ +module F +let x = 1 +""" + "G.fs", + """ +module G +let x = 1 +""" + ] + |> FProject.Make CompileOutput.Library + + let all = [ encodeDecodeSimple @@ -213,6 +254,7 @@ let d (c: CType) = fsFsi emptyNamespace dependentSignatures + fullyParallel ] type Case = From d14ba17c6c0913fe802f16882d9847fa07fadc76 Mon Sep 17 00:00:00 2001 From: janusz Date: Wed, 23 Nov 2022 18:35:32 +0000 Subject: [PATCH 19/21] changes - try to optimize env merging by keeping deltas --- src/Compiler/Driver/OptimizeInputs.fs | 58 +++++++++---- src/Compiler/Driver/OptimizeInputs.fsi | 4 +- src/Compiler/Driver/OptimizeTypes.fs | 11 ++- src/Compiler/Driver/ParseAndCheckInputs.fs | 2 + src/Compiler/Driver/ParseAndCheckInputs.fsi | 2 + src/Compiler/Driver/fsc.fs | 83 ++++++++++++------- src/Compiler/Optimize/Optimizer.fs | 33 +++++++- src/Compiler/Optimize/Optimizer.fsi | 4 + src/Compiler/TypedTree/TypedTreeOps.fsi | 2 + .../Code/GraphBasedOpt.fs | 46 +++++++--- .../Code/GraphProcessing.fs | 3 + .../Code/Parallel.fs | 7 +- .../Code/ParallelTypeChecking.fs | 39 ++++++++- tests/ParallelTypeCheckingTests/Program.fs | 10 ++- .../Tests/AssemblySetUp.fs | 3 +- .../Tests/TestCompilation.fs | 17 +++- .../Tests/TestCompilationFromCmdlineArgs.fs | 20 +++++ 17 files changed, 268 insertions(+), 76 deletions(-) diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index c2fd81a1c7..6cb5ef10d9 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -74,7 +74,7 @@ let collectResults (inputs: CollectorInputs) : CollectorOutputs = let (optEnvPhase1, _, _, _), _ = phase1 optEnvPhase1 - files, lastFilePhase1Env + files, lastFilePhase1Env.Full type Phase = | Phase1 @@ -179,8 +179,8 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu let env, _, hidingInfo = previous |> Option.map getPhase1Res - |> Option.defaultValue (env0, file, hidingInfo0) - let inputs = env, hidingInfo, file + |> Option.defaultValue ({Delta=env0; Full=env0}, file, {Delta=hidingInfo0; Full=hidingInfo0}) + let inputs = env.Full, hidingInfo.Full, file let phase1Res = phase1 inputs res.Phase1 <- Some phase1Res @@ -194,14 +194,16 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu | Phase.Phase2 -> // take env from previous file - let env, _ = + let env = previous |> Option.map getPhase2Res - |> Option.defaultValue (env0, file) + |> Option.map fst + |> Option.map (fun daf -> daf.Full) + |> Option.defaultValue env0 let _optEnv, file, hidingInfo = res |> getPhase1Res - let inputs = env, hidingInfo, file + let inputs = env, hidingInfo.Full, file let phase2Res = phase2 inputs res.Phase2 <- Some phase2Res @@ -219,14 +221,16 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu let env = previous |> Option.map getPhase3Res + |> Option.map (fun daf -> daf.Full) |> Option.defaultValue env0 // impl file let _, file = res |> getPhase2Res - let _phase1Env, _, hidingInfo = + let hidingInfo = res |> getPhase1Res + |> fun (a,b,c) -> c.Full let inputs = env, hidingInfo, file let phase3Res = phase3 inputs res.Phase3 <- Some phase3Res @@ -260,7 +264,7 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu let collected = results |> collectResults collected -type Goer = IReadOnlyDictionary -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs +type Goer = IReadOnlyDictionary -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs let mutable goer: Goer option = None @@ -270,7 +274,7 @@ type OptimizerMode = | PartiallyParallel | GraphBased -let mutable OptimizerMode: OptimizerMode = OptimizerMode.Sequential +let mutable optimizerMode: OptimizerMode = OptimizerMode.Sequential let ApplyAllOptimizations ( @@ -317,9 +321,21 @@ let ApplyAllOptimizations let env0 = optEnv0 + let envToDaf (env: IncrementalOptimizationEnv) (env0: IncrementalOptimizationEnv) = + { + Full = env + Delta = subtractEnv env env0 + } + + let hiddenToDaf (env: SignatureHidingInfo) (env0: SignatureHidingInfo) = + { + Full = env + Delta = subtractHidingInfo env env0 + } + let phase1 (env: IncrementalOptimizationEnv, hidden: SignatureHidingInfo, implFile: CheckedImplFile) : Phase1Res = //ReportTime tcConfig ("Initial simplify") - Optimizer.OptimizeImplFile( + let (a,b,c,d), e = Optimizer.OptimizeImplFile( optSettings, ccu, tcGlobals, @@ -332,6 +348,9 @@ let ApplyAllOptimizations hidden, implFile ) + let a = envToDaf a env + let d = hiddenToDaf d hidden + (a,b,c,d),e let phase2 (env: IncrementalOptimizationEnv, hidden: SignatureHidingInfo, implFile: CheckedImplFile) : Phase2Res = let implFile = LowerLocalMutables.TransformImplFile tcGlobals importMap implFile @@ -369,9 +388,9 @@ let ApplyAllOptimizations ) //PrintWholeAssemblyImplementation tcConfig outfile (sprintf "extra-loop-%d" n) implFile - optEnvExtraLoop, implFile + envToDaf optEnvExtraLoop env, implFile else - env, implFile + {Delta = IncrementalOptimizationEnv.Empty; Full = env}, implFile let phase3 (env: IncrementalOptimizationEnv, hidden: SignatureHidingInfo, implFile: CheckedImplFile) : Phase3Res = // Only do this on the first pass! @@ -418,16 +437,19 @@ let ApplyAllOptimizations ) //PrintWholeAssemblyImplementation tcConfig outfile "post-rec-opt" implFile - optEnvFinalSimplify, implFile + envToDaf optEnvFinalSimplify env, implFile else - env, implFile + {Delta = IncrementalOptimizationEnv.Empty; Full = env}, implFile let results, optEnvFirstLoop = - match OptimizerMode with + match optimizerMode with | OptimizerMode.GraphBased -> - let graph = - [||] - |> readOnlyDict + let graph = ParseAndCheckInputs.graph + graph + |> Seq.iter (fun (KeyValue(f, deps)) -> + let d = System.String.Join(",", deps) + printfn $"{f} - {d}" + ) let goer = goer.Value let a, b = goer graph env0 (phase1, phase2, phase3) (implFiles |> List.toArray) diff --git a/src/Compiler/Driver/OptimizeInputs.fsi b/src/Compiler/Driver/OptimizeInputs.fsi index 6552eb4e4e..b03bed636d 100644 --- a/src/Compiler/Driver/OptimizeInputs.fsi +++ b/src/Compiler/Driver/OptimizeInputs.fsi @@ -53,7 +53,7 @@ val NormalizeAssemblyRefs: CompilationThreadToken * ILGlobals * TcImports -> (IL val GetGeneratedILModuleName: CompilerTarget -> string -> string type FilePhaseFuncs = Phase1Fun * Phase2Fun * Phase3Fun -type Goer = IReadOnlyDictionary -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs +type Goer = IReadOnlyDictionary -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs val mutable goer: Goer option type OptimizerMode = @@ -61,4 +61,4 @@ type OptimizerMode = | PartiallyParallel | GraphBased -val mutable OptimizerMode: OptimizerMode \ No newline at end of file +val mutable optimizerMode: OptimizerMode \ No newline at end of file diff --git a/src/Compiler/Driver/OptimizeTypes.fs b/src/Compiler/Driver/OptimizeTypes.fs index e2f15dd6ba..d37024406c 100644 --- a/src/Compiler/Driver/OptimizeTypes.fs +++ b/src/Compiler/Driver/OptimizeTypes.fs @@ -6,10 +6,15 @@ open FSharp.Compiler.TcGlobals open FSharp.Compiler.TypedTree open FSharp.Compiler.TypedTreeOps +type DeltaAndFull<'a> = + { + Delta: 'a + Full: 'a + } type OptimizeDuringCodeGen = bool -> Expr -> Expr type OptimizeRes = - (IncrementalOptimizationEnv * CheckedImplFile * ImplFileOptimizationInfo * SignatureHidingInfo) * OptimizeDuringCodeGen + (DeltaAndFull * CheckedImplFile * ImplFileOptimizationInfo * DeltaAndFull) * OptimizeDuringCodeGen type Optimize = OptimizationSettings * @@ -32,11 +37,11 @@ type Phase1Res = OptimizeRes type Phase1Fun = Phase1Inputs -> Phase1Res type Phase2Inputs = PhaseInputs -type Phase2Res = IncrementalOptimizationEnv * CheckedImplFile +type Phase2Res = DeltaAndFull * CheckedImplFile type Phase2Fun = Phase2Inputs -> Phase2Res type Phase3Inputs = PhaseInputs -type Phase3Res = IncrementalOptimizationEnv * CheckedImplFile +type Phase3Res = DeltaAndFull * CheckedImplFile type Phase3Fun = Phase3Inputs -> Phase3Res type Phase = diff --git a/src/Compiler/Driver/ParseAndCheckInputs.fs b/src/Compiler/Driver/ParseAndCheckInputs.fs index 5d1322efca..02eec82077 100644 --- a/src/Compiler/Driver/ParseAndCheckInputs.fs +++ b/src/Compiler/Driver/ParseAndCheckInputs.fs @@ -1755,3 +1755,5 @@ let CheckClosedInputSet (ctok, checkForErrors, tcConfig: TcConfig, tcImports, tc tcState.Ccu.Deref.Contents <- ccuContents tcState, topAttrs, declaredImpls, tcEnvAtEndOfLastFile + +let mutable graph: System.Collections.Generic.IReadOnlyDictionary = null \ No newline at end of file diff --git a/src/Compiler/Driver/ParseAndCheckInputs.fsi b/src/Compiler/Driver/ParseAndCheckInputs.fsi index 708111d89f..5ccaa14a33 100644 --- a/src/Compiler/Driver/ParseAndCheckInputs.fsi +++ b/src/Compiler/Driver/ParseAndCheckInputs.fsi @@ -228,3 +228,5 @@ val CheckOneInputAndFinish: tcState: TcState * input: ParsedInput -> Cancellable<(TcEnv * TopAttribs * CheckedImplFile list * ModuleOrNamespaceType list) * TcState> + +val mutable graph: System.Collections.Generic.IReadOnlyDictionary \ No newline at end of file diff --git a/src/Compiler/Driver/fsc.fs b/src/Compiler/Driver/fsc.fs index a3968cef5e..66edbe55ab 100644 --- a/src/Compiler/Driver/fsc.fs +++ b/src/Compiler/Driver/fsc.fs @@ -20,6 +20,7 @@ open System.Reflection open System.Text open System.Threading +open System.Threading.Tasks open Internal.Utilities open Internal.Utilities.Library open Internal.Utilities.Library.Extras @@ -823,13 +824,6 @@ let main3 ReportTime tcConfig "Encode Interface Data" let exportRemapping = MakeExportRemapping generatedCcu generatedCcu.Contents - let sigDataAttributes, sigDataResources = - try - EncodeSignatureData(tcConfig, tcGlobals, exportRemapping, generatedCcu, outfile, false) - with e -> - errorRecoveryNoRange e - exiter.Exit 1 - let metadataVersion = match tcConfig.metadataVersion with | Some v -> v @@ -837,35 +831,60 @@ let main3 match frameworkTcImports.DllTable.TryFind tcConfig.primaryAssembly.Name with | Some ib -> ib.RawMetadata.TryGetILModuleDef().Value.MetadataVersion | _ -> "" + + let (sigDataAttributes, sigDataResources), optimizedImpls, optDataResources = + let mutable sigDataAttributes: ILAttribute list = [] + let mutable sigDataResources : ILResource list = [] + let a1 = + async { + try + let sigDataAttributes2, sigDataResources2 = EncodeSignatureData(tcConfig, tcGlobals, exportRemapping, generatedCcu, outfile, false) + sigDataAttributes <- sigDataAttributes2 + sigDataResources <- sigDataResources2 + with e -> + errorRecoveryNoRange e + exiter.Exit 1 + } - let optimizedImpls, optDataResources = - // Perform optimization - use _ = UseBuildPhase BuildPhase.Optimize - - let optEnv0 = GetInitialOptimizationEnv(tcImports, tcGlobals) - - let importMap = tcImports.GetImportMap() - - let optimizedImpls, optimizationData, _ = - ApplyAllOptimizations( - tcConfig, - tcGlobals, - (LightweightTcValForUsingInBuildMethodCall tcGlobals), - outfile, - importMap, - false, - optEnv0, - generatedCcu, - typedImplFiles - ) - - AbortOnError(diagnosticsLogger, exiter) + let mutable optimizedImpls2 : CheckedAssemblyAfterOptimization option = None + let mutable optDataResources : ILResource list = [] + let a2 = + async { + // Perform optimization + use _ = UseBuildPhase BuildPhase.Optimize + + let optEnv0 = GetInitialOptimizationEnv(tcImports, tcGlobals) + + let importMap = tcImports.GetImportMap() + + let optimizedImpls, optimizationData, _ = + ApplyAllOptimizations( + tcConfig, + tcGlobals, + (LightweightTcValForUsingInBuildMethodCall tcGlobals), + outfile, + importMap, + false, + optEnv0, + generatedCcu, + typedImplFiles + ) - // Encode the optimization data - ReportTime tcConfig ("Encoding OptData") + AbortOnError(diagnosticsLogger, exiter) - optimizedImpls, EncodeOptimizationData(tcGlobals, tcConfig, outfile, exportRemapping, (generatedCcu, optimizationData), false) + // Encode the optimization data + ReportTime tcConfig ("Encoding OptData") + optimizedImpls2 <- Some optimizedImpls + optDataResources <- EncodeOptimizationData(tcGlobals, tcConfig, outfile, exportRemapping, (generatedCcu, optimizationData), false) + } + + let t1 = a1 |> Async.StartAsTask + let t2 = a2 |> Async.StartAsTask + t1.Wait() + t2.Wait() + (sigDataAttributes, sigDataResources), optimizedImpls2.Value, optDataResources + // Pass on only the minimum information required for the next phase Args( ctok, diff --git a/src/Compiler/Optimize/Optimizer.fs b/src/Compiler/Optimize/Optimizer.fs index b2b1338d10..e3a9866c50 100644 --- a/src/Compiler/Optimize/Optimizer.fs +++ b/src/Compiler/Optimize/Optimizer.fs @@ -487,14 +487,45 @@ let mergeMaps<'Key, 'Value when 'Key : comparison> (maps: Map<'Key, 'Value>[]) = |> Array.collect Map.toArray |> Map.ofArray +let mapDiff<'a,'b when 'a: comparison> (a: Map<'a,'b>) (b: Map<'a,'b>) : Map<'a,'b> = + a + // TODO What if a & b contain a key but with different values? Is that possible? + |> Seq.choose (fun (KeyValue(k, v)) -> + match a.TryFind k with + | Some _ -> None + | None -> Some (k, v) + ) + |> Map.ofSeq + +let subtractEnv (a: IncrementalOptimizationEnv) (b: IncrementalOptimizationEnv): IncrementalOptimizationEnv = + { + a with + //latestBoundId = env0.latestBoundId // Not used across files + dontInline = Zset.diff a.dontInline b.dontInline// sum + typarInfos = a.typarInfos |> List.skip b.typarInfos.Length // sum + //functionVal = None // Not used across files + dontSplitVars = ValMap(mapDiff a.dontSplitVars.Contents b.dontSplitVars.Contents) // sum + //disableMethodSplitting = false // not used across files + localExternalVals = mapDiff a.localExternalVals b.localExternalVals // sum + globalModuleInfos = mapDiff a.globalModuleInfos b.globalModuleInfos // sum + //methEnv = { pipelineCount = 0 } // Not used across files + } +let subtractHidingInfo (a: SignatureHidingInfo) (b: SignatureHidingInfo) : SignatureHidingInfo = + { + SignatureHidingInfo.HiddenTycons = Zset.diff a.HiddenTycons b.HiddenTycons + HiddenTyconReprs = Zset.diff a.HiddenTyconReprs b.HiddenTyconReprs + HiddenVals = Zset.diff a.HiddenVals b.HiddenVals + HiddenRecdFields = Zset.diff a.HiddenRecdFields b.HiddenRecdFields + HiddenUnionCases = Zset.diff a.HiddenUnionCases b.HiddenUnionCases + } let mergeEnvs (env0: IncrementalOptimizationEnv) (envs: IncrementalOptimizationEnv[]): IncrementalOptimizationEnv = let envs = Array.append [|env0|] envs // TODO use a single HashSet for perf? let dontInline = envs - |> Array.collect (fun e -> e.dontInline |> Zset.elements |> List.toArray) + |> Array.collect (fun e -> e.dontInline.ToArray()) |> fun xs -> Zset.Create (Int64.order, xs) let typarInfos = envs |> Array.toList |> List.collect (fun e -> e.typarInfos) let dontSplitVars = diff --git a/src/Compiler/Optimize/Optimizer.fsi b/src/Compiler/Optimize/Optimizer.fsi index 95b65a5602..d57de0f702 100644 --- a/src/Compiler/Optimize/Optimizer.fsi +++ b/src/Compiler/Optimize/Optimizer.fsi @@ -63,6 +63,10 @@ type IncrementalOptimizationEnv = val mergeEnvs : IncrementalOptimizationEnv -> IncrementalOptimizationEnv[] -> IncrementalOptimizationEnv +val subtractEnv : IncrementalOptimizationEnv -> IncrementalOptimizationEnv -> IncrementalOptimizationEnv + +val subtractHidingInfo : SignatureHidingInfo -> SignatureHidingInfo -> SignatureHidingInfo + /// For building optimization environments incrementally val internal BindCcu: CcuThunk -> CcuOptimizationInfo -> IncrementalOptimizationEnv -> TcGlobals -> IncrementalOptimizationEnv diff --git a/src/Compiler/TypedTree/TypedTreeOps.fsi b/src/Compiler/TypedTree/TypedTreeOps.fsi index fa6ae8d2b2..aeebd18033 100755 --- a/src/Compiler/TypedTree/TypedTreeOps.fsi +++ b/src/Compiler/TypedTree/TypedTreeOps.fsi @@ -407,6 +407,8 @@ val mkExprAddrOfExpr: [] type ValMap<'T> = + new : StampMap<'T> -> ValMap<'T> + member Contents: StampMap<'T> member Item: Val -> 'T with get diff --git a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs index 6898621db3..be52947e4a 100644 --- a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs +++ b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs @@ -2,6 +2,7 @@ #nowarn "1182" +open System.Collections.Generic open System.IO open FSharp.Compiler.Optimizer open FSharp.Compiler.Service.Driver.OptimizeTypes @@ -33,7 +34,7 @@ let collectResults (inputs: CollectorInputs) : CollectorOutputs = let (optEnvPhase1, _, _, _), _ = phase1 optEnvPhase1 - files, lastFilePhase1Env + files, lastFilePhase1Env.Full type FilePhaseFuncs = Phase1Fun * Phase2Fun * Phase3Fun type FileResults = @@ -104,11 +105,24 @@ let mergeHidingInfos (empty: SignatureHidingInfo) (infos: SignatureHidingInfo[]) type Goer = IdxGraph -> IncrementalOptimizationEnv -> FilePhaseFuncs -> CheckedImplFile[] -> CollectorOutputs -let goGraph (idxGraph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFuncs) (files: CheckedImplFile[]) : CollectorOutputs = +let goGraph (idxGraph: IReadOnlyDictionary) (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFuncs) (files: CheckedImplFile[]) : CollectorOutputs = // Create a 3x graph by cloning each file with its deps for each phase. Add links from phase3 -> phase2 -> phase1 + let idxGraph = + idxGraph + // Temporary to workaround a bug in graph creation + |> Seq.map (fun (KeyValue(f, deps)) -> + f, deps |> Array.filter (fun d -> d <> f) + ) + |> readOnlyDict let graph = idxGraph |> Seq.collect (fun (KeyValue(file, deps)) -> + let deps = + deps + // Temporary to workaround a bug in graph creation + |> Array.filter (fun d -> d <> file) + |> Array.map FileIdx + let file = FileIdx file // Create a node per each phase Phase.all |> Array.map (fun phase -> @@ -128,6 +142,12 @@ let goGraph (idxGraph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, ph ) |> readOnlyDict + graph + |> Seq.iter (fun (KeyValue(f, deps)) -> + let d = System.String.Join(",", deps) + printfn $"{f} ==> {d}" + ) + let mergeEnvs envs = mergeEnvs env0 envs @@ -137,12 +157,12 @@ let goGraph (idxGraph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, ph let results = Array.init files.Length (fun _ -> FileResults.Empty) - let getRes (FileIdx idx) = results[idx] + let getRes idx = results[idx] let hidingInfo0 = SignatureHidingInfo.Empty let work (x: Node) : unit = - let {Idx=idx; Phase=phase} = x - let file = files[idx.Idx] + let {Idx=FileIdx idx; Phase=phase} = x + let file = files[idx] let res = getRes idx let depResults = transitiveIdxGraph[idx] @@ -155,14 +175,14 @@ let goGraph (idxGraph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, ph depResults |> Array.map (fun r -> let (a,_b,_c,_d), _e = r.Get1() - a + a.Delta ) |> mergeEnvs let hidingInfo = depResults |> Array.map (fun r -> let (_a,_b,_c,d), _e = r.Get1() - d + d.Delta ) |> mergeHidingInfos hidingInfo0 let inputs = env, hidingInfo, file @@ -174,12 +194,12 @@ let goGraph (idxGraph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, ph depResults |> Array.map (fun r -> let a,_b = r.Get2() - a + a.Delta ) |> mergeEnvs // Get file and hidingInfo from phase1 of the current file let (_optEnv, file, _, hidingInfo), _ = res.Get1() - let inputs = env, hidingInfo, file + let inputs = env, hidingInfo.Full, file let phase2Res = phase2 inputs res.Phase2 <- Some phase2Res | Phase.Phase3 -> @@ -188,21 +208,23 @@ let goGraph (idxGraph: IdxGraph) (env0: IncrementalOptimizationEnv) ((phase1, ph depResults |> Array.map (fun r -> let a,_b = r.Get3() - a + a.Delta ) |> mergeEnvs // Get file and hidingInfo from phase1 of the current file let (_optEnv, _, _, hidingInfo), _ = res.Get1() // Get impl file from phase2 let _, file = res.Get2() - let inputs = env, hidingInfo, file + let inputs = env, hidingInfo.Full, file let phase3Res = phase3 inputs res.Phase3 <- Some phase3Res + + printfn $"{x} finished" GraphProcessing.processGraphSimpler graph work - 1 + 12 let completeResults = results |> Array.map FileResults.complete let collected = collectResults completeResults diff --git a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs index 1a0382eeff..01a87310ed 100644 --- a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs +++ b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs @@ -428,7 +428,9 @@ let processGraphSimpler<'Item when 'Item: equality and 'Item: comparison> : Node3<'Item>[] = let _deps = lookupMany node.Info.Deps + printfn $"{node.Info.Item} DoWork" doWork node.Info.Item + printfn $"{node.Info.Item} DoneWork" // Need to double-check that only one dependency schedules this dependant let unblocked = node.Info.Dependants @@ -441,6 +443,7 @@ let processGraphSimpler<'Item when 'Item: equality and 'Item: comparison> x.ProcessedDepsCount) pdc = x.Info.Deps.Length ) + printfn $"{node.Info.Item} unblocked gathered" unblocked use cts = new CancellationTokenSource() diff --git a/tests/ParallelTypeCheckingTests/Code/Parallel.fs b/tests/ParallelTypeCheckingTests/Code/Parallel.fs index 33df5fb267..91f626a611 100644 --- a/tests/ParallelTypeCheckingTests/Code/Parallel.fs +++ b/tests/ParallelTypeCheckingTests/Code/Parallel.fs @@ -24,18 +24,21 @@ let processInParallel let mutable processedCount = 0 let processItem item = - printfn $"Processing {_itemToString item}" + printfn $"{_itemToString item} Processing" let toSchedule = work item + printfn $"{_itemToString item} worked" let processedCount = lock processedCountLock (fun () -> processedCount <- processedCount + 1 processedCount) + + printfn $"{_itemToString item} after lock" let toScheduleString = toSchedule |> Array.map _itemToString |> (fun names -> String.Join(", ", names)) - printfn $"Scheduling {toSchedule.Length} items: {toScheduleString}" + printfn $"{_itemToString item} finished - scheduling {toSchedule.Length} items: {toScheduleString}" toSchedule |> Array.iter bc.Add processedCount diff --git a/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs b/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs index 9e85287172..064069977c 100644 --- a/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs +++ b/tests/ParallelTypeCheckingTests/Code/ParallelTypeChecking.fs @@ -59,8 +59,43 @@ let CheckMultipleInputsInParallel |> List.map (fun ast -> ast.FileName, ast) |> readOnlyDict |> ConcurrentDictionary<_, _> - + + let fileMap = + sourceFiles + |> Array.map (fun f -> f.AST.FileName, f.Idx.Idx) + |> Map.ofArray + + let convertDep (dep: int) = + match sourceFiles[dep].AST with + | AST.ImplFile _ -> dep + | AST.SigFile _ -> + let sigFile = sourceFiles[dep].AST.FileName + let implFile = sigFile.TrimEnd('i') + fileMap[implFile] + let graph = DepResolving.DependencyResolution.detectFileDependencies sourceFiles + ParseAndCheckInputs.graph <- + let implIndices = + inputs + |> List.mapi (fun i x -> i, x) + |> List.choose (fun (i, ast) -> + match ast with + | AST.SigFile _ -> None + | AST.ImplFile _ -> Some i + ) + |> List.mapi (fun i idx -> idx, i) + |> readOnlyDict + + let implGraph = + graph.Graph + |> Seq.map (fun (KeyValue(f, deps)) -> f, deps) + |> Seq.map (fun (f, deps) -> f.Idx.Idx, deps |> Array.map (fun f -> convertDep f.Idx.Idx)) + |> Seq.filter (fun (f, _) -> implIndices.ContainsKey f) + |> Seq.map (fun (f, deps) -> + f |> fun x -> implIndices[x], deps |> Array.filter implIndices.ContainsKey |> Array.map (fun x -> implIndices[x]) + ) + |> readOnlyDict + implGraph let mutable nextIdx = (graph.Files |> Array.map (fun f -> f.File.Idx.Idx) |> Array.max) + 1 @@ -261,6 +296,6 @@ let CheckMultipleInputsInParallel (fun file -> file.Idx.Idx) state (fun it -> not <| it.Name.EndsWith(".fsix")) - 10 + 12 partialResults |> Array.toList, tcState) diff --git a/tests/ParallelTypeCheckingTests/Program.fs b/tests/ParallelTypeCheckingTests/Program.fs index 190a046bec..3488ab58b2 100644 --- a/tests/ParallelTypeCheckingTests/Program.fs +++ b/tests/ParallelTypeCheckingTests/Program.fs @@ -4,6 +4,7 @@ open FSharp.Compiler open FSharp.Compiler.CompilerConfig +open FSharp.Compiler.OptimizeInputs open ParallelTypeCheckingTests.TestCompilation open ParallelTypeCheckingTests.TestUtils @@ -34,7 +35,14 @@ open ParallelTypeCheckingTests.TestCompilationFromCmdlineArgs let main _argv = ParseAndCheckInputs.CheckMultipleInputsUsingGraphMode <- ParallelTypeChecking.CheckMultipleInputsInParallel - OptimizeInputs.UseParallelOptimizer <- bool.Parse(_argv[0]) + FSharp.Compiler.OptimizeInputs.goer <- ParallelTypeCheckingTests.Code.GraphBasedOpt.goGraph |> Some + let mode = + match _argv[0] with + | "graph" -> OptimizerMode.GraphBased + | "sequential" -> OptimizerMode.Sequential + | "partial" -> OptimizerMode.PartiallyParallel + | _ -> failwith $"unknown mode {_argv[0]}" + OptimizeInputs.optimizerMode <- mode // let args = _parse _argv // let args = { args with LineLimit = None } let componentTests = codebases[System.Int32.Parse(_argv[1])] diff --git a/tests/ParallelTypeCheckingTests/Tests/AssemblySetUp.fs b/tests/ParallelTypeCheckingTests/Tests/AssemblySetUp.fs index 351ea9d55c..9a538b4942 100644 --- a/tests/ParallelTypeCheckingTests/Tests/AssemblySetUp.fs +++ b/tests/ParallelTypeCheckingTests/Tests/AssemblySetUp.fs @@ -3,6 +3,7 @@ open NUnit.Framework open OpenTelemetry.Trace + /// One-time setup for NUnit tests [] type AssemblySetUp() = @@ -12,7 +13,7 @@ type AssemblySetUp() = member this.SetUp() = FSharp.Compiler.ParseAndCheckInputs.CheckMultipleInputsUsingGraphMode <- ParallelTypeCheckingTests.ParallelTypeChecking.CheckMultipleInputsInParallel - + FSharp.Compiler.OptimizeInputs.goer <- ParallelTypeCheckingTests.Code.GraphBasedOpt.goGraph |> Some tracerProvider <- ParallelTypeCheckingTests.TestUtils.setupOtel () |> Some [] diff --git a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs index 36e2752265..2e2625e807 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TestCompilation.fs @@ -1,8 +1,10 @@ module ParallelTypeCheckingTests.TestCompilation +open FSharp.Compiler.OptimizeInputs open FSharp.Test open FSharp.Test.Compiler open NUnit.Framework +open ParallelTypeCheckingTests.Code open ParallelTypeCheckingTests.TestUtils open FSharp.Compiler @@ -298,7 +300,7 @@ let codebases = Codebases.all [] let ``Compile a valid project using graph-based type-checking`` (project: FProject) = - global.FSharp.Compiler.OptimizeInputs.UseParallelOptimizer <- false + global.FSharp.Compiler.OptimizeInputs.optimizerMode <- OptimizerMode.Sequential compileAValidProject { Method = Method.Graph @@ -307,7 +309,18 @@ let ``Compile a valid project using graph-based type-checking`` (project: FProje [] let ``Compile a valid project using graph-based type-checking, parallel opt`` (project: FProject) = - global.FSharp.Compiler.OptimizeInputs.UseParallelOptimizer <- true + global.FSharp.Compiler.OptimizeInputs.optimizerMode <- OptimizerMode.PartiallyParallel + compileAValidProject + { + Method = Method.Graph + Project = project + } + + +[] +let ``Compile a valid project using graph-based type-checking, graph opt`` (project: FProject) = + global.FSharp.Compiler.OptimizeInputs.optimizerMode <- OptimizerMode.GraphBased + OptimizeInputs.goer <- GraphBasedOpt.goGraph |> Some compileAValidProject { Method = Method.Graph diff --git a/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs b/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs index 109e47e64c..ca3cd13527 100644 --- a/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs +++ b/tests/ParallelTypeCheckingTests/Tests/TestCompilationFromCmdlineArgs.fs @@ -2,10 +2,12 @@ open FSharp.Compiler.CompilerConfig open FSharp.Compiler.DiagnosticsLogger +open FSharp.Compiler.OptimizeInputs open NUnit.Framework open System open FSharp.Compiler open ParallelTypeCheckingTests +open ParallelTypeCheckingTests.Code open ParallelTypeCheckingTests.TestUtils type Codebase = @@ -91,6 +93,7 @@ let internal codebaseToConfig code method = [] [] let ``1. Test sequential type-checking`` (code: Codebase) = + OptimizeInputs.optimizerMode <- OptimizerMode.Sequential let config = codebaseToConfig code Method.Sequential TestCompilerFromArgs config @@ -98,6 +101,7 @@ let ``1. Test sequential type-checking`` (code: Codebase) = [] [] let ``2. Test parallelfs type-checking`` (code: Codebase) = + OptimizeInputs.optimizerMode <- OptimizerMode.Sequential let config = codebaseToConfig code Method.ParallelCheckingOfBackedImplFiles TestCompilerFromArgs config @@ -105,4 +109,20 @@ let ``2. Test parallelfs type-checking`` (code: Codebase) = [] let ``3. Test graph-based type-checking`` (code: Codebase) = let config = codebaseToConfig code Method.Graph + OptimizeInputs.optimizerMode <- OptimizerMode.Sequential TestCompilerFromArgs config + +/// Before running this test, you must prepare the codebase by running the script 'FCS.prepare.ps1' +[] +let ``4. Test graph-based type-checking - partially parallel opt`` (code: Codebase) = + let config = codebaseToConfig code Method.Graph + OptimizeInputs.optimizerMode <- OptimizerMode.PartiallyParallel + TestCompilerFromArgs config + +/// Before running this test, you must prepare the codebase by running the script 'FCS.prepare.ps1' +[] +let ``4. Test graph-based type-checking - graph-based opt`` (code: Codebase) = + let config = codebaseToConfig code Method.Graph + OptimizeInputs.optimizerMode <- OptimizerMode.GraphBased + TestCompilerFromArgs config + OptimizeInputs.goer <- Some GraphBasedOpt.goGraph \ No newline at end of file From 9f5c483a65dca36e2fbb9a7fa3fb2cbae7e9eee7 Mon Sep 17 00:00:00 2001 From: janusz Date: Wed, 23 Nov 2022 19:09:14 +0000 Subject: [PATCH 20/21] changes --- src/Compiler/Driver/OptimizeInputs.fs | 2 +- src/Compiler/Optimize/Optimizer.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index 6cb5ef10d9..73f08df738 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -230,7 +230,7 @@ let go (env0: IncrementalOptimizationEnv) ((phase1, phase2, phase3): FilePhaseFu let hidingInfo = res |> getPhase1Res - |> fun (a,b,c) -> c.Full + |> fun (_a,_b,c) -> c.Full let inputs = env, hidingInfo, file let phase3Res = phase3 inputs res.Phase3 <- Some phase3Res diff --git a/src/Compiler/Optimize/Optimizer.fs b/src/Compiler/Optimize/Optimizer.fs index e3a9866c50..8a313f52e7 100644 --- a/src/Compiler/Optimize/Optimizer.fs +++ b/src/Compiler/Optimize/Optimizer.fs @@ -491,7 +491,7 @@ let mapDiff<'a,'b when 'a: comparison> (a: Map<'a,'b>) (b: Map<'a,'b>) : Map<'a, a // TODO What if a & b contain a key but with different values? Is that possible? |> Seq.choose (fun (KeyValue(k, v)) -> - match a.TryFind k with + match b.TryFind k with | Some _ -> None | None -> Some (k, v) ) From 678e738c660bcd840e25438a438f8e48f1a9c926 Mon Sep 17 00:00:00 2001 From: janusz Date: Wed, 23 Nov 2022 20:17:45 +0000 Subject: [PATCH 21/21] changes --- src/Compiler/Driver/OptimizeInputs.fs | 10 +++--- .../Code/GraphBasedOpt.fs | 14 ++++----- .../Code/GraphProcessing.fs | 8 ++--- .../Code/Parallel.fs | 12 +++---- tests/ParallelTypeCheckingTests/Program.fs | 31 ++++++++++--------- 5 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index 73f08df738..f1ee94e2ff 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -445,11 +445,11 @@ let ApplyAllOptimizations match optimizerMode with | OptimizerMode.GraphBased -> let graph = ParseAndCheckInputs.graph - graph - |> Seq.iter (fun (KeyValue(f, deps)) -> - let d = System.String.Join(",", deps) - printfn $"{f} - {d}" - ) + // graph + // |> Seq.iter (fun (KeyValue(f, deps)) -> + // let d = System.String.Join(",", deps) + // printfn $"{f} - {d}" + // ) let goer = goer.Value let a, b = goer graph env0 (phase1, phase2, phase3) (implFiles |> List.toArray) diff --git a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs index be52947e4a..cc1a6b5b18 100644 --- a/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs +++ b/tests/ParallelTypeCheckingTests/Code/GraphBasedOpt.fs @@ -141,12 +141,12 @@ let goGraph (idxGraph: IReadOnlyDictionary) (env0: IncrementalOptimi ) ) |> readOnlyDict - - graph - |> Seq.iter (fun (KeyValue(f, deps)) -> - let d = System.String.Join(",", deps) - printfn $"{f} ==> {d}" - ) + // + // graph + // |> Seq.iter (fun (KeyValue(f, deps)) -> + // let d = System.String.Join(",", deps) + // printfn $"{f} ==> {d}" + // ) let mergeEnvs envs = mergeEnvs env0 envs @@ -219,7 +219,7 @@ let goGraph (idxGraph: IReadOnlyDictionary) (env0: IncrementalOptimi let phase3Res = phase3 inputs res.Phase3 <- Some phase3Res - printfn $"{x} finished" + // printfn $"{x} finished" GraphProcessing.processGraphSimpler graph diff --git a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs index 01a87310ed..7b20b7983c 100644 --- a/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs +++ b/tests/ParallelTypeCheckingTests/Code/GraphProcessing.fs @@ -421,16 +421,16 @@ let processGraphSimpler<'Item when 'Item: equality and 'Item: comparison> |> Seq.filter (fun n -> n.Info.Deps.Length = 0) |> Seq.toArray - printfn $"Node count: {nodes.Count}" + // printfn $"Node count: {nodes.Count}" let work (node: Node3<'Item>) : Node3<'Item>[] = let _deps = lookupMany node.Info.Deps - printfn $"{node.Info.Item} DoWork" + // printfn $"{node.Info.Item} DoWork" doWork node.Info.Item - printfn $"{node.Info.Item} DoneWork" + // printfn $"{node.Info.Item} DoneWork" // Need to double-check that only one dependency schedules this dependant let unblocked = node.Info.Dependants @@ -443,7 +443,7 @@ let processGraphSimpler<'Item when 'Item: equality and 'Item: comparison> x.ProcessedDepsCount) pdc = x.Info.Deps.Length ) - printfn $"{node.Info.Item} unblocked gathered" + // printfn $"{node.Info.Item} unblocked gathered" unblocked use cts = new CancellationTokenSource() diff --git a/tests/ParallelTypeCheckingTests/Code/Parallel.fs b/tests/ParallelTypeCheckingTests/Code/Parallel.fs index 91f626a611..fa0e9b80fa 100644 --- a/tests/ParallelTypeCheckingTests/Code/Parallel.fs +++ b/tests/ParallelTypeCheckingTests/Code/Parallel.fs @@ -24,21 +24,21 @@ let processInParallel let mutable processedCount = 0 let processItem item = - printfn $"{_itemToString item} Processing" + // printfn $"{_itemToString item} Processing" let toSchedule = work item - printfn $"{_itemToString item} worked" + // printfn $"{_itemToString item} worked" let processedCount = lock processedCountLock (fun () -> processedCount <- processedCount + 1 processedCount) - printfn $"{_itemToString item} after lock" + // printfn $"{_itemToString item} after lock" - let toScheduleString = - toSchedule |> Array.map _itemToString |> (fun names -> String.Join(", ", names)) + // let toScheduleString = + // toSchedule |> Array.map _itemToString |> (fun names -> String.Join(", ", names)) - printfn $"{_itemToString item} finished - scheduling {toSchedule.Length} items: {toScheduleString}" + // printfn $"{_itemToString item} finished - scheduling {toSchedule.Length} items: {toScheduleString}" toSchedule |> Array.iter bc.Add processedCount diff --git a/tests/ParallelTypeCheckingTests/Program.fs b/tests/ParallelTypeCheckingTests/Program.fs index 3488ab58b2..57ff4c1fa9 100644 --- a/tests/ParallelTypeCheckingTests/Program.fs +++ b/tests/ParallelTypeCheckingTests/Program.fs @@ -33,20 +33,21 @@ let _parse (argv: string[]) : Args = open ParallelTypeCheckingTests.TestCompilationFromCmdlineArgs [] let main _argv = - ParseAndCheckInputs.CheckMultipleInputsUsingGraphMode <- - ParallelTypeChecking.CheckMultipleInputsInParallel - FSharp.Compiler.OptimizeInputs.goer <- ParallelTypeCheckingTests.Code.GraphBasedOpt.goGraph |> Some - let mode = - match _argv[0] with - | "graph" -> OptimizerMode.GraphBased - | "sequential" -> OptimizerMode.Sequential - | "partial" -> OptimizerMode.PartiallyParallel - | _ -> failwith $"unknown mode {_argv[0]}" - OptimizeInputs.optimizerMode <- mode - // let args = _parse _argv - // let args = { args with LineLimit = None } - let componentTests = codebases[System.Int32.Parse(_argv[1])] - let config = codebaseToConfig componentTests Method.Graph - TestCompilerFromArgs config + for _i in [1;2] do + ParseAndCheckInputs.CheckMultipleInputsUsingGraphMode <- + ParallelTypeChecking.CheckMultipleInputsInParallel + FSharp.Compiler.OptimizeInputs.goer <- ParallelTypeCheckingTests.Code.GraphBasedOpt.goGraph |> Some + let mode = + match _argv[0] with + | "graph" -> OptimizerMode.GraphBased + | "sequential" -> OptimizerMode.Sequential + | "partial" -> OptimizerMode.PartiallyParallel + | _ -> failwith $"unknown mode {_argv[0]}" + OptimizeInputs.optimizerMode <- mode + // let args = _parse _argv + // let args = { args with LineLimit = None } + let componentTests = codebases[System.Int32.Parse(_argv[1])] + let config = codebaseToConfig componentTests Method.Graph + TestCompilerFromArgs config 0