[stdlib] Consistency tweaks for Unsafe[Mutable][Raw]BufferPointer

stdlib/public/core/Hasher.swift

@@ -227,9 +227,7 @@ extension Hasher {
           remaining -= c
         }
       }
-      _internalInvariant(
-        remaining == 0 ||
-        Int(bitPattern: data) & (MemoryLayout<UInt64>.alignment - 1) == 0)
+      _internalInvariant(remaining == 0 || data._isWellAligned(for: UInt64.self))
 
       // Load as many aligned words as there are in the input buffer
       while remaining >= MemoryLayout<UInt64>.size {

stdlib/public/core/Span/MutableSpan.swift

@@ -72,9 +72,8 @@ extension MutableSpan where Element: ~Copyable {
     _unsafeElements buffer: UnsafeMutableBufferPointer<Element>
   ) {
     _precondition(
-      ((Int(bitPattern: buffer.baseAddress) &
-        (MemoryLayout<Element>.alignment &- 1)) == 0),
-      "baseAddress must be properly aligned to access Element"
+      buffer._isWellAligned(for: Element.self),
+      "buffer must be properly aligned to access Element"
     )
     let ms = unsafe MutableSpan<Element>(_unchecked: buffer)
     self = unsafe _overrideLifetime(ms, borrowing: buffer)
@@ -121,9 +120,8 @@ extension MutableSpan where Element: BitwiseCopyable {
     _unsafeBytes buffer: UnsafeMutableRawBufferPointer
   ) {
     _precondition(
-      ((Int(bitPattern: buffer.baseAddress) &
-        (MemoryLayout<Element>.alignment &- 1)) == 0),
-      "baseAddress must be properly aligned to access Element"
+      buffer._isWellAligned(for: Element.self),
+      "buffer must be properly aligned to access Element"
     )
     let (byteCount, stride) = (buffer.count, MemoryLayout<Element>.stride)
     let (count, remainder) = byteCount.quotientAndRemainder(dividingBy: stride)

stdlib/public/core/Span/Span.swift

@@ -108,14 +108,11 @@ extension Span where Element: ~Copyable {
   public init(
     _unsafeElements buffer: UnsafeBufferPointer<Element>
   ) {
-    //FIXME: Workaround for https://github.com/swiftlang/swift/issues/77235
-    let baseAddress = unsafe UnsafeRawPointer(buffer.baseAddress)
     _precondition(
-      ((Int(bitPattern: baseAddress) &
-        (MemoryLayout<Element>.alignment &- 1)) == 0),
-      "baseAddress must be properly aligned to access Element"
+      buffer._isWellAligned(for: Element.self),
+      "buffer must be properly aligned to access Element"
     )
-    let span = unsafe Span(_unchecked: baseAddress, count: buffer.count)
+    let span = unsafe Span(_unchecked: buffer.baseAddress, count: buffer.count)
     // As a trivial value, 'baseAddress' does not formally depend on the
     // lifetime of 'buffer'. Make the dependence explicit.
     self = unsafe _overrideLifetime(span, borrowing: buffer)
@@ -237,19 +234,16 @@ extension Span where Element: BitwiseCopyable {
   public init(
     _unsafeBytes buffer: UnsafeRawBufferPointer
   ) {
-    //FIXME: Workaround for https://github.com/swiftlang/swift/issues/77235
-    let baseAddress = buffer.baseAddress
     _precondition(
-      ((Int(bitPattern: baseAddress) &
-        (MemoryLayout<Element>.alignment &- 1)) == 0),
-      "baseAddress must be properly aligned to access Element"
+      buffer._isWellAligned(for: Element.self),
+      "buffer must be properly aligned to access Element"
     )
     let (byteCount, stride) = (buffer.count, MemoryLayout<Element>.stride)
     let (count, remainder) = byteCount.quotientAndRemainder(dividingBy: stride)
     _precondition(
       remainder == 0, "Span must contain a whole number of elements"
     )
-    let span = unsafe Span(_unchecked: baseAddress, count: count)
+    let span = unsafe Span(_unchecked: buffer.baseAddress, count: count)
     // As a trivial value, 'baseAddress' does not formally depend on the
     // lifetime of 'buffer'. Make the dependence explicit.
     self = unsafe _overrideLifetime(span, borrowing: buffer)

stdlib/public/core/UnsafeBufferPointer.swift.gyb

@@ -780,6 +780,22 @@ extension Unsafe${Mutable}BufferPointer:
 %  end
 }
 
+extension Unsafe${Mutable}BufferPointer where Element: ~Copyable {
+  @safe
+  @_alwaysEmitIntoClient
+  public func _isWellAligned() -> Bool {
+    guard let p = baseAddress else { return true }
+    return p._isWellAligned()
+  }
+
+  @safe
+  @_alwaysEmitIntoClient
+  public func _isWellAligned<T: ~Copyable>(for: T.Type) -> Bool {
+    guard let p = baseAddress else { return true }
+    return p._isWellAligned(for: T.self)
+  }
+}
+
 extension Unsafe${Mutable}BufferPointer {
 %  if not Mutable:
   /// Creates a buffer over the same memory as the given buffer slice.
@@ -1404,7 +1420,7 @@ extension Unsafe${Mutable}BufferPointer where Element: ~Copyable {
     }
 
     _debugPrecondition(
-      unsafe Int(bitPattern: .init(base)) & (MemoryLayout<T>.alignment-1) == 0,
+      _isWellAligned(for: T.self),
       "baseAddress must be a properly aligned pointer for types Element and T"
     )
 

stdlib/public/core/UnsafePointer.swift

@@ -400,8 +400,8 @@ extension UnsafePointer where Pointee: ~Copyable {
     capacity count: Int,
     _ body: (_ pointer: UnsafePointer<T>) throws(E) -> Result
   ) throws(E) -> Result {
-    unsafe _debugPrecondition(
-      Int(bitPattern: .init(_rawValue)) & (MemoryLayout<T>.alignment-1) == 0 &&
+    _debugPrecondition(
+      _isWellAligned(for: T.self) &&
       ( count == 1 ||
         ( MemoryLayout<Pointee>.stride > MemoryLayout<T>.stride
           ? MemoryLayout<Pointee>.stride % MemoryLayout<T>.stride == 0
@@ -410,6 +410,19 @@ extension UnsafePointer where Pointee: ~Copyable {
       ),
       "self must be a properly aligned pointer for types Pointee and T"
     )
+    return try unsafe _uncheckedWithMemoryRebound(
+      to: type, capacity: count, body)
+  }
+
+  @_alwaysEmitIntoClient
+  @_transparent
+  internal func _uncheckedWithMemoryRebound<
+    T: ~Copyable, E: Error, Result: ~Copyable
+  >(
+    to type: T.Type,
+    capacity count: Int,
+    _ body: (_ pointer: UnsafePointer<T>) throws(E) -> Result
+  ) throws(E) -> Result {
     let binding = Builtin.bindMemory(_rawValue, count._builtinWordValue, T.self)
     defer { Builtin.rebindMemory(_rawValue, binding) }
     return try unsafe body(.init(_rawValue))
@@ -469,6 +482,19 @@ extension UnsafePointer where Pointee: ~Copyable {
   }
 }
 
+extension UnsafePointer where Pointee: ~Copyable {
+  @safe
+  @_alwaysEmitIntoClient
+  public func _isWellAligned() -> Bool {
+    (Int(bitPattern: self) & (MemoryLayout<Pointee>.alignment &- 1)) == 0
+  }
+
+  @safe
+  @_alwaysEmitIntoClient
+  internal func _isWellAligned<T: ~Copyable>(for: T.Type) -> Bool {
+    UnsafeRawPointer(_rawValue)._isWellAligned(for: T.self)
+  }
+}
 
 /// A pointer for accessing and manipulating data of a
 /// specific type.
@@ -1248,8 +1274,8 @@ extension UnsafeMutablePointer where Pointee: ~Copyable {
     capacity count: Int,
     _ body: (_ pointer: UnsafeMutablePointer<T>) throws(E) -> Result
   ) throws(E) -> Result {
-    unsafe _debugPrecondition(
-      Int(bitPattern: .init(_rawValue)) & (MemoryLayout<T>.alignment-1) == 0 &&
+    _debugPrecondition(
+      _isWellAligned(for: T.self) &&
       ( count == 1 ||
         ( MemoryLayout<Pointee>.stride > MemoryLayout<T>.stride
           ? MemoryLayout<Pointee>.stride % MemoryLayout<T>.stride == 0
@@ -1382,3 +1408,17 @@ extension UnsafeMutablePointer where Pointee: ~Copyable {
     )._unsafelyUnwrappedUnchecked
   }
 }
+
+extension UnsafeMutablePointer where Pointee: ~Copyable {
+  @safe
+  @_alwaysEmitIntoClient
+  public func _isWellAligned() -> Bool {
+    unsafe UnsafePointer(self)._isWellAligned()
+  }
+
+  @safe
+  @_alwaysEmitIntoClient
+  internal func _isWellAligned<T: ~Copyable>(for: T.Type) -> Bool {
+    UnsafeRawPointer(_rawValue)._isWellAligned(for: T.self)
+  }
+}

stdlib/public/core/UnsafeRawBufferPointer.swift.gyb

@@ -100,6 +100,19 @@ public struct Unsafe${Mutable}RawBufferPointer {
   @usableFromInline
   internal let _position, _end: Unsafe${Mutable}RawPointer?
 
+  // This works around _debugPrecondition() impacting the performance of
+  // optimized code. (rdar://72246338)
+  @_alwaysEmitIntoClient
+  internal init(
+    @_nonEphemeral _uncheckedStart start: Unsafe${Mutable}RawPointer?,
+    count: Int
+  ) {
+    _internalInvariant(count >= 0)
+    _internalInvariant(unsafe count == 0 || start != nil)
+    unsafe _position = start
+    unsafe _end = start.map { unsafe $0 + _assumeNonNegative(count) }
+  }
+
   /// Creates a buffer over the specified number of contiguous bytes starting
   /// at the given pointer.
   ///
@@ -116,9 +129,14 @@ public struct Unsafe${Mutable}RawBufferPointer {
     _debugPrecondition(count >= 0, "${Self} with negative count")
     _debugPrecondition(unsafe count == 0 || start != nil,
       "${Self} has a nil start and nonzero count")
+    unsafe self.init(_uncheckedStart: start, count: count)
+  }
 
-    unsafe _position = start
-    unsafe _end = start.map { unsafe $0 + _assumeNonNegative(count) }
+  @safe
+  @_alwaysEmitIntoClient
+  public init(_empty: ()) {
+    unsafe _position = nil
+    unsafe _end = nil
   }
 }
 
@@ -816,7 +834,7 @@ extension Unsafe${Mutable}RawBufferPointer {
     }  
 
     _debugPrecondition(
-      Int(bitPattern: base) & (MemoryLayout<S.Element>.alignment-1) == 0,
+      _isWellAligned(for: S.Element.self),
       "buffer base address must be properly aligned to access S.Element"
     )
 
@@ -875,7 +893,7 @@ extension Unsafe${Mutable}RawBufferPointer {
         return unsafe .init(start: nil, count: 0)
       }
       _debugPrecondition(
-        Int(bitPattern: baseAddress) & (MemoryLayout<C.Element>.alignment-1) == 0,
+        _isWellAligned(for: C.Element.self),
         "buffer base address must be properly aligned to access C.Element"
       )
       _precondition(
@@ -900,7 +918,7 @@ extension Unsafe${Mutable}RawBufferPointer {
     }
     _internalInvariant(unsafe _end != nil)
     _debugPrecondition(
-      Int(bitPattern: baseAddress) & (MemoryLayout<C.Element>.alignment-1) == 0,
+      _isWellAligned(for: C.Element.self),
       "buffer base address must be properly aligned to access C.Element"
     )
     var iterator = source.makeIterator()
@@ -961,7 +979,7 @@ extension Unsafe${Mutable}RawBufferPointer {
       return unsafe .init(start: nil, count: 0)
     }
     _debugPrecondition(
-      Int(bitPattern: baseAddress) & (MemoryLayout<T>.alignment-1) == 0,
+      _isWellAligned(for: T.self),
       "buffer base address must be properly aligned to access T"
     )
     _precondition(
@@ -1103,7 +1121,7 @@ extension Unsafe${Mutable}RawBufferPointer {
       return try unsafe body(.init(start: nil, count: 0))
     }
     _debugPrecondition(
-      Int(bitPattern: s) & (MemoryLayout<T>.alignment-1) == 0,
+      _isWellAligned(for: T.self),
       "baseAddress must be a properly aligned pointer for type T"
     )
     // initializer ensures _end is nil only when _position is nil.
@@ -1115,6 +1133,13 @@ extension Unsafe${Mutable}RawBufferPointer {
     return try unsafe body(.init(start: .init(s._rawValue), count: n))
   }
 
+  @safe
+  @_alwaysEmitIntoClient
+  public func _isWellAligned<T: ~Copyable>(for type: T.Type) -> Bool {
+    guard let s = baseAddress else { return true }
+    return s._isWellAligned(for: T.self)
+  }
+
   /// Returns a typed buffer to the memory referenced by this buffer,
   /// assuming that the memory is already bound to the specified type.
   ///

stdlib/public/core/UnsafeRawPointer.swift

@@ -400,7 +400,7 @@ extension UnsafeRawPointer {
     _ body: (_ pointer: UnsafePointer<T>) throws(E) -> Result
   ) throws(E) -> Result {
     _debugPrecondition(
-      Int(bitPattern: self) & (MemoryLayout<T>.alignment-1) == 0,
+      _isWellAligned(for: T.self),
       "self must be a properly aligned pointer for type T"
     )
     let binding = Builtin.bindMemory(_rawValue, count._builtinWordValue, T.self)
@@ -446,8 +446,8 @@ extension UnsafeRawPointer {
     fromByteOffset offset: Int = 0,
     as type: T.Type
   ) -> T {
-    unsafe _debugPrecondition(0 == (UInt(bitPattern: self + offset)
-        & (UInt(MemoryLayout<T>.alignment) - 1)),
+    unsafe _debugPrecondition(
+      (self + offset)._isWellAligned(for: T.self),
       "load from misaligned raw pointer")
 
     let rawPointer = unsafe (self + offset)._rawValue
@@ -584,6 +584,12 @@ extension UnsafeRawPointer {
     return .init(Builtin.inttoptr_Word(bits._builtinWordValue))
   }
 
+  @safe
+  @_alwaysEmitIntoClient
+  public func _isWellAligned<T: ~Copyable>(for type: T.Type) -> Bool {
+    unsafe self == self.alignedDown(for: type)
+  }
+
   /// Obtain the next pointer whose bit pattern is a multiple of `alignment`.
   ///
   /// If the bit pattern of `self` is a multiple of `alignment`,
@@ -1017,7 +1023,7 @@ extension UnsafeMutableRawPointer {
     _ body: (_ pointer: UnsafeMutablePointer<T>) throws(E) -> Result
   ) throws(E) -> Result {
     _debugPrecondition(
-      Int(bitPattern: self) & (MemoryLayout<T>.alignment-1) == 0,
+      _isWellAligned(for: T.self),
       "self must be a properly aligned pointer for type T"
     )
     let binding = Builtin.bindMemory(_rawValue, count._builtinWordValue, T.self)
@@ -1285,8 +1291,8 @@ extension UnsafeMutableRawPointer {
     fromByteOffset offset: Int = 0,
     as type: T.Type
   ) -> T {
-    unsafe _debugPrecondition(0 == (UInt(bitPattern: self + offset)
-        & (UInt(MemoryLayout<T>.alignment) - 1)),
+    unsafe _debugPrecondition(
+      (self + offset)._isWellAligned(for: T.self),
       "load from misaligned raw pointer")
 
     let rawPointer = unsafe (self + offset)._rawValue
@@ -1499,8 +1505,8 @@ extension UnsafeMutableRawPointer {
   internal func _legacy_se0349_storeBytes_internal<T>(
     of value: T, toByteOffset offset: Int = 0, as type: T.Type
   ) {
-    unsafe _debugPrecondition(0 == (UInt(bitPattern: self + offset)
-        & (UInt(MemoryLayout<T>.alignment) - 1)),
+    unsafe _debugPrecondition(
+      (self + offset)._isWellAligned(for: T.self),
       "storeBytes to misaligned raw pointer")
 
     var temp = value
@@ -1588,6 +1594,12 @@ extension UnsafeMutableRawPointer {
     return .init(Builtin.inttoptr_Word(bits._builtinWordValue))
   }
 
+  @safe
+  @_alwaysEmitIntoClient
+  public func _isWellAligned<T: ~Copyable>(for type: T.Type) -> Bool {
+    UnsafeRawPointer(self)._isWellAligned(for: type)
+  }
+
   /// Obtain the next pointer whose bit pattern is a multiple of `alignment`.
   ///
   /// If the bit pattern of `self` is a multiple of `alignment`,