Merge branch 'main' into windows-ci

Author: compnerd

Runtimes/Core/core/CMakeLists.txt

@@ -80,7 +80,6 @@ add_library(swiftCore
   FlatMap.swift
   Flatten.swift
   FloatingPoint.swift
-  FloatingPointToString.swift
   Hashable.swift
   AnyHashable.swift # ORDER DEPENDENCY
   Hasher.swift

Runtimes/Core/runtime/CMakeLists.txt

@@ -52,6 +52,7 @@ add_library(swiftRuntime OBJECT
     RefCount.cpp
     ReflectionMirror.cpp
     RuntimeInvocationsTracking.cpp
+    SwiftDtoa.cpp
     SwiftTLSContext.cpp
     ThreadingError.cpp
     Tracing.cpp

SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift

@@ -504,7 +504,8 @@ private extension ApplySite {
     {
       if self is FullApplySite,
          // If the function is inlined later, there is no point in specializing it.
-         !callee.shouldOptimize || callee.inlineStrategy == .always
+         !callee.shouldOptimize || callee.inlineStrategy == .heuristicAlways ||
+         callee.inlineStrategy == .always
       {
         return nil
       }

SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ConstantCapturePropagation.swift

@@ -66,6 +66,10 @@ let constantCapturePropagation = FunctionPass(name: "constant-capture-propagatio
       continue
     }
 
+    if !context.continueWithNextSubpassRun(for: partialApply) {
+      return
+    }
+
     optimizeClosureWithDeadCaptures(of: partialApply, context)
 
     if partialApply.isDeleted {
@@ -301,9 +305,15 @@ private extension PartialApplyInst {
     var nonConstArgs = [Operand]()
     var hasKeypath = false
     for argOp in argumentOperands {
-      if argOp.value.isConstant(hasKeypath: &hasKeypath) {
+      // In non-OSSA we don't know where to insert the compensating release for a propagated keypath.
+      // Therefore bail if a keypath has multiple uses.
+      switch argOp.value.isConstant(requireSingleUse: !parentFunction.hasOwnership && !isOnStack) {
+      case .constant:
+        constArgs.append(argOp)
+      case .constantWithKeypath:
         constArgs.append(argOp)
-      } else {
+        hasKeypath = true
+      case .notConstant:
         nonConstArgs.append(argOp)
       }
     }
@@ -333,33 +343,49 @@ private extension FullApplySite {
   }
 }
 
+private enum ConstantKind {
+  case notConstant
+  case constant
+  case constantWithKeypath
+
+  func merge(with other: ConstantKind) -> ConstantKind {
+    switch (self, other) {
+      case (.notConstant, _):      return .notConstant
+      case (_, .notConstant):      return .notConstant
+      case (.constant, .constant): return .constant
+      default:                     return .constantWithKeypath
+    }
+  }
+}
+
 private extension Value {
-  func isConstant(hasKeypath: inout Bool) -> Bool {
+  func isConstant(requireSingleUse: Bool) -> ConstantKind {
     // All instructions handled here must also be handled in
     // `FunctionSignatureSpecializationMangler::mangleConstantProp`.
+    let result: ConstantKind
     switch self {
     case let si as StructInst:
-      for op in si.operands {
-        if !op.value.isConstant(hasKeypath: &hasKeypath) {
-          return false
-        }
-      }
-      return true
+      result = si.operands.reduce(.constant, {
+        $0.merge(with: $1.value.isConstant(requireSingleUse: requireSingleUse))
+      })
     case is ThinToThickFunctionInst, is ConvertFunctionInst, is UpcastInst, is OpenExistentialRefInst:
-      return (self as! UnaryInstruction).operand.value.isConstant(hasKeypath: &hasKeypath)
+      result = (self as! UnaryInstruction).operand.value.isConstant(requireSingleUse: requireSingleUse)
     case is StringLiteralInst, is IntegerLiteralInst, is FloatLiteralInst, is FunctionRefInst, is GlobalAddrInst:
-      return true
+      result = .constant
     case let keyPath as KeyPathInst:
-      hasKeypath = true
       guard keyPath.operands.isEmpty,
             keyPath.hasPattern,
             !keyPath.substitutionMap.hasAnySubstitutableParams
       else {
-        return false
+        return .notConstant
       }
-      return true
+      result = .constantWithKeypath
     default:
-      return false
+      return .notConstant
+    }
+    if requireSingleUse, result == .constantWithKeypath, !uses.ignoreDebugUses.isSingleUse {
+      return .notConstant
     }
+    return result
   }
 }

SwiftCompilerSources/Sources/Optimizer/FunctionPasses/DeadStoreElimination.swift

@@ -167,8 +167,15 @@ private extension StoreInst {
 
   var hasValidOwnershipForDeadStoreElimination: Bool {
     switch storeOwnership {
-    case .unqualified, .trivial:
+    case .unqualified:
       return true
+    case .trivial:
+      // Storing a trivial enum case in a non-trivial enum must be treated like a non-trivial
+      // init or assign, e.g.
+      //   %1 = enum $Optional<String>, #Optional.none!enumelt
+      //   store %1 to [trivial] %0  // <- cannot delete this store!
+      //   store %2 to [assign] %0
+      return source.type.isTrivial(in: parentFunction)
     case .initialize, .assign:
       // In OSSA, non-trivial values cannot be dead-store eliminated because that could shrink
       // the lifetime of the original stored value (because it's not kept in memory anymore).

SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift

@@ -693,6 +693,12 @@ private extension MovableInstructions {
     accessPath: AccessPath,
     context: FunctionPassContext
   ) -> Bool {
+
+    // If the memory is not initialized at all exits, it would be wrong to insert stores at exit blocks.
+    guard memoryIsInitializedAtAllExits(of: loop, accessPath: accessPath, context) else {
+      return false
+    }
+
     // Initially load the value in the loop pre header.
     let builder = Builder(before: loop.preheader!.terminator, context)
     var firstStore: StoreInst?
@@ -721,7 +727,18 @@ private extension MovableInstructions {
     guard let firstStore else {
       return false
     }
-    
+
+    // We currently don't support split `load [take]`, i.e. `load [take]` which does _not_ load all
+    // non-trivial fields of the initial value.
+    for case let load as LoadInst in loadsAndStores {
+      if load.loadOwnership == .take,
+         let path = accessPath.getProjection(to: load.address.accessPath),
+         !firstStore.destination.type.isProjectingEntireNonTrivialMembers(path: path, in: load.parentFunction)
+      {
+        return false
+      }
+    }
+
     var ssaUpdater = SSAUpdater(
       function: firstStore.parentFunction,
       type: firstStore.destination.type.objectType,
@@ -792,7 +809,13 @@ private extension MovableInstructions {
       let rootVal = currentVal ?? ssaUpdater.getValue(inMiddleOf: block)
 
       if loadInst.operand.value.accessPath == accessPath {
-        loadInst.replace(with: rootVal, context)
+        if loadInst.loadOwnership == .copy {
+          let builder = Builder(before: loadInst, context)
+          let copy = builder.createCopyValue(operand: rootVal)
+          loadInst.replace(with: copy, context)
+        } else {
+          loadInst.replace(with: rootVal, context)
+        }
         changed = true
         continue
       }
@@ -802,7 +825,11 @@ private extension MovableInstructions {
       }
 
       let builder = Builder(before: loadInst, context)
-      let projection = rootVal.createProjection(path: projectionPath, builder: builder)
+      let projection = if loadInst.loadOwnership == .copy {
+        rootVal.createProjectionAndCopy(path: projectionPath, builder: builder)
+      } else {
+        rootVal.createProjection(path: projectionPath, builder: builder)
+      }
       loadInst.replace(with: projection, context)
 
       changed = true
@@ -831,6 +858,69 @@ private extension MovableInstructions {
 
     return changed
   }
+
+  func memoryIsInitializedAtAllExits(of loop: Loop, accessPath: AccessPath, _ context: FunctionPassContext) -> Bool {
+
+    // Perform a simple dataflow analysis which checks if there is a path from a `load [take]`
+    // (= the only kind of instruction which can de-initialize the memory) to a loop exit without
+    // a `store` in between.
+
+    var stores = InstructionSet(context)
+    defer { stores.deinitialize() }
+    for case let store as StoreInst in loadsAndStores where store.storesTo(accessPath) {
+      stores.insert(store)
+    }
+
+    var exitInsts = InstructionSet(context)
+    defer { exitInsts.deinitialize() }
+    exitInsts.insert(contentsOf: loop.exitBlocks.lazy.map { $0.instructions.first! })
+
+    var worklist = InstructionWorklist(context)
+    defer { worklist.deinitialize() }
+    for case let load as LoadInst in loadsAndStores where load.loadOwnership == .take && load.loadsFrom(accessPath) {
+      worklist.pushIfNotVisited(load)
+    }
+
+    while let inst = worklist.pop() {
+      if stores.contains(inst) {
+        continue
+      }
+      if exitInsts.contains(inst) {
+        return false
+      }
+      worklist.pushSuccessors(of: inst)
+    }
+    return true
+  }
+}
+
+private extension Type {
+  func isProjectingEntireNonTrivialMembers(path: SmallProjectionPath, in function: Function) -> Bool {
+    let (kind, index, subPath) = path.pop()
+    switch kind {
+    case .root:
+      return true
+    case .structField:
+      guard let fields = getNominalFields(in: function) else {
+        return false
+      }
+      for (fieldIdx, fieldType) in fields.enumerated() {
+        if fieldIdx != index && !fieldType.isTrivial(in: function) {
+          return false
+        }
+      }
+      return fields[index].isProjectingEntireNonTrivialMembers(path: subPath, in: function)
+    case .tupleField:
+      for (elementIdx, elementType) in tupleElements.enumerated() {
+        if elementIdx != index && !elementType.isTrivial(in: function) {
+          return false
+        }
+      }
+      return tupleElements[index].isProjectingEntireNonTrivialMembers(path: subPath, in: function)
+    default:
+      fatalError("path is not materializable")
+    }
+  }
 }
 
 private extension Instruction {

SwiftCompilerSources/Sources/Optimizer/ModulePasses/MandatoryPerformanceOptimizations.swift

@@ -302,7 +302,9 @@ private func shouldInline(apply: FullApplySite, callee: Function, alreadyInlined
     return false
   }
 
-  if apply.parentFunction.isGlobalInitOnceFunction && callee.inlineStrategy == .always {
+  if apply.parentFunction.isGlobalInitOnceFunction && (
+      callee.inlineStrategy == .heuristicAlways ||
+      callee.inlineStrategy == .always) {
     // Some arithmetic operations, like integer conversions, are not transparent but `inline(__always)`.
     // Force inlining them in global initializers so that it's possible to statically initialize the global.
     return true

SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift

@@ -96,23 +96,26 @@ extension MutatingContext {
   func inlineFunction(apply: FullApplySite, mandatoryInline: Bool) {
     // This is only a best-effort attempt to notify the new cloned instructions as changed.
     // TODO: get a list of cloned instructions from the `inlineFunction`
-    let instAfterInling: Instruction?
+    let instBeforeInlining = apply.previous
+    let instAfterInlining: Instruction?
     switch apply {
     case is ApplyInst:
-      instAfterInling = apply.next
+      instAfterInlining = apply.next
     case let beginApply as BeginApplyInst:
       let next = beginApply.next!
-      instAfterInling = (next is EndApplyInst ? nil : next)
+      instAfterInlining = (next is EndApplyInst ? nil : next)
     case is TryApplyInst:
-      instAfterInling = apply.parentBlock.next?.instructions.first
+      instAfterInlining = apply.parentBlock.next?.instructions.first
     default:
-      instAfterInling = nil
+      instAfterInlining = nil
     }
 
     bridgedPassContext.inlineFunction(apply.bridged, mandatoryInline)
 
-    if let instAfterInling = instAfterInling {
-      notifyNewInstructions(from: apply, to: instAfterInling)
+    if let instBeforeInlining = instBeforeInlining?.next,
+       let instAfterInlining = instAfterInlining,
+       !instAfterInlining.isDeleted {
+      notifyNewInstructions(from: instBeforeInlining, to: instAfterInlining)
     }
   }
 

SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift

@@ -125,20 +125,32 @@ extension Value {
     return true
   }
 
+  /// Project out a sub-field of this value according to `path`.
+  /// If this is an "owned" value the result is an "owned" value which forwards the original value.
+  /// This only works if _all_ non-trivial fields are projected. Otherwise some non-trivial results of
+  /// `destructure_struct` or `destructure_tuple` will be leaked.
   func createProjection(path: SmallProjectionPath, builder: Builder) -> Value {
     let (kind, index, subPath) = path.pop()
+    let result: Value
     switch kind {
     case .root:
       return self
     case .structField:
-      let structExtract = builder.createStructExtract(struct: self, fieldIndex: index)
-      return structExtract.createProjection(path: subPath, builder: builder)
+      if ownership == .owned {
+        result = builder.createDestructureStruct(struct: self).results[index]
+      } else {
+        result = builder.createStructExtract(struct: self, fieldIndex: index)
+      }
     case .tupleField:
-      let tupleExtract = builder.createTupleExtract(tuple: self, elementIndex: index)
-      return tupleExtract.createProjection(path: subPath, builder: builder)
+      if ownership == .owned {
+        result = builder.createDestructureTuple(tuple: self).results[index]
+      } else {
+        result = builder.createTupleExtract(tuple: self, elementIndex: index)
+      }
     default:
       fatalError("path is not materializable")
     }
+    return result.createProjection(path: subPath, builder: builder)
   }
 
   func createAddressProjection(path: SmallProjectionPath, builder: Builder) -> Value {

SwiftCompilerSources/Sources/SIL/DataStructures/Worklist.swift

@@ -77,7 +77,7 @@ public typealias ValueWorklist = Worklist<ValueSet>
 public typealias OperandWorklist = Worklist<OperandSet>
 
 extension InstructionWorklist {
-  public mutating func pushPredecessors(of inst: Instruction, ignoring ignoreInst: Instruction) {
+  public mutating func pushPredecessors(of inst: Instruction, ignoring ignoreInst: Instruction? = nil) {
     if let prev = inst.previous {
       if prev != ignoreInst {
         pushIfNotVisited(prev)
@@ -92,7 +92,7 @@ extension InstructionWorklist {
     }
   }
 
-  public mutating func pushSuccessors(of inst: Instruction, ignoring ignoreInst: Instruction) {
+  public mutating func pushSuccessors(of inst: Instruction, ignoring ignoreInst: Instruction? = nil) {
     if let succ = inst.next {
       if succ != ignoreInst {
         pushIfNotVisited(succ)