UTF8Span (#78531)

Add support for UTF8Span

Also, refactor validation and grapheme breaking

Author: milseman

Runtimes/Core/core/CMakeLists.txt

@@ -208,6 +208,13 @@ add_library(swiftCore
   UnsafeRawPointer.swift
   UTFEncoding.swift
   UTF8.swift
+  UTF8EncodingError.swift
+  UTF8Span.swift
+  UTF8SpanBits.swift
+  UTF8SpanComparisons.swift
+  UTF8SpanFundamentals.swift
+  UTF8SpanInternalHelpers.swift
+  UTF8SpanIterators.swift
   UTF16.swift
   UTF32.swift
   Unicode.swift # ORDER DEPENDENCY: must follow new unicode support

stdlib/public/core/CMakeLists.txt

@@ -214,6 +214,13 @@ split_embedded_sources(
   EMBEDDED UnsafeRawPointer.swift
   EMBEDDED UTFEncoding.swift
   EMBEDDED UTF8.swift
+  EMBEDDED UTF8EncodingError.swift
+  EMBEDDED UTF8Span.swift
+  EMBEDDED UTF8SpanBits.swift
+  EMBEDDED UTF8SpanComparisons.swift
+  EMBEDDED UTF8SpanFundamentals.swift
+  EMBEDDED UTF8SpanInternalHelpers.swift
+  EMBEDDED UTF8SpanIterators.swift
   EMBEDDED UTF16.swift
   EMBEDDED UTF32.swift
   EMBEDDED Unicode.swift # ORDER DEPENDENCY: must follow new unicode support

stdlib/public/core/GroupInfo.json

@@ -205,6 +205,15 @@
     "RawSpan.swift",
     "Span.swift"
   ],
+  "UTF8Span": [ 
+    "UTF8EncodingError.swift",
+    "UTF8Span.swift",
+    "UTF8SpanBits.swift",
+    "UTF8SpanComparisons.swift",
+    "UTF8SpanFundamentals.swift",
+    "UTF8SpanInternalHelpers.swift",
+    "UTF8SpanIterators.swift"
+  ],
   "Protocols": [
     "CompilerProtocols.swift",
     "ShadowProtocols.swift"

stdlib/public/core/String.swift

@@ -1112,108 +1112,4 @@ extension String {
   }
 }
 
-extension _StringGutsSlice {
-  internal func _isScalarNFCQC(
-    _ scalar: Unicode.Scalar,
-    _ prevCCC: inout UInt8
-  ) -> Bool {
-    let normData = Unicode._NormData(scalar, fastUpperbound: 0x300)
 
-    if prevCCC > normData.ccc, normData.ccc != 0 {
-      return false
-    }
-
-    if !normData.isNFCQC {
-      return false
-    }
-
-    prevCCC = normData.ccc
-    return true
-  }
-
-  internal func _withNFCCodeUnits(_ f: (UInt8) throws -> Void) rethrows {
-    let substring = String(_guts)[range]
-    // Fast path: If we're already NFC (or ASCII), then we don't need to do
-    // anything at all.
-    if _fastPath(_guts.isNFC) {
-      try substring.utf8.forEach(f)
-      return
-    }
-
-    var isNFCQC = true
-    var prevCCC: UInt8 = 0
-
-    if _guts.isFastUTF8 {
-      _fastNFCCheck(&isNFCQC, &prevCCC)
-
-      // Because we have access to the fastUTF8, we can go through that instead
-      // of accessing the UTF8 view on String.
-      if isNFCQC {
-        try unsafe withFastUTF8 {
-          for unsafe byte in unsafe $0 {
-            try f(byte)
-          }
-        }
-
-        return
-      }
-    } else {
-      for scalar in substring.unicodeScalars {
-        if !_isScalarNFCQC(scalar, &prevCCC) {
-          isNFCQC = false
-          break
-        }
-      }
-
-      if isNFCQC {
-        for byte in substring.utf8 {
-          try f(byte)
-        }
-
-        return
-      }
-    }
-
-    for scalar in substring.unicodeScalars._internalNFC {
-      try scalar.withUTF8CodeUnits {
-        for unsafe byte in unsafe $0 {
-          try f(byte)
-        }
-      }
-    }
-  }
-
-  internal func _fastNFCCheck(_ isNFCQC: inout Bool, _ prevCCC: inout UInt8) {
-    unsafe withFastUTF8 { utf8 in
-      var position = 0
-
-      while position < utf8.count {
-        // If our first byte is less than 0xCC, then it means we're under the
-        // 0x300 scalar value and everything up to 0x300 is NFC already.
-        if unsafe utf8[position] < 0xCC {
-          // If our first byte is less than 0xC0, then it means it is ASCII
-          // and only takes up a single byte.
-          if unsafe utf8[position] < 0xC0 {
-            position &+= 1
-          } else {
-            // Otherwise, this is a 2 byte < 0x300 sequence.
-            position &+= 2
-          }
-          // ASCII always has ccc of 0.
-          prevCCC = 0
-
-          continue
-        }
-
-        let (scalar, len) = unsafe _decodeScalar(utf8, startingAt: position)
-
-        if !_isScalarNFCQC(scalar, &prevCCC) {
-          isNFCQC = false
-          return
-        }
-
-        position &+= len
-      }
-    }
-  }
-}

stdlib/public/core/StringComparison.swift

@@ -97,7 +97,7 @@ internal func _stringCompareInternal(
 }
 
 @_effects(readonly)
-private func _stringCompareFastUTF8(
+internal func _stringCompareFastUTF8(
   _ utf8Left: UnsafeBufferPointer<UInt8>,
   _ utf8Right: UnsafeBufferPointer<UInt8>,
   expecting: _StringComparisonResult,

stdlib/public/core/StringCreate.swift

@@ -117,7 +117,7 @@ extension String {
       return unsafe (String._uncheckedFromUTF8(
         input, asciiPreScanResult: extraInfo.isASCII
       ), false)
-    case .error(let initialRange):
+    case .error(_, let initialRange):
         return unsafe (repairUTF8(input, firstKnownBrokenRange: initialRange), true)
     }
   }
@@ -139,7 +139,7 @@ extension String {
         newIsASCII: info.isASCII
       )
       return result.asString
-    case .error(let initialRange):
+    case .error(_, let initialRange):
       defer { _fixLifetime(result) }
       //This could be optimized to use excess tail capacity
       return unsafe repairUTF8(result.codeUnits, firstKnownBrokenRange: initialRange)