From 208740eeadec23c586d4973ca6b2686ed3415a1d Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Thu, 30 Mar 2023 09:35:55 -0600 Subject: [PATCH 01/10] Align availability macro with OS versions --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 5d45950db..f02ef1828 100644 --- a/Package.swift +++ b/Package.swift @@ -7,7 +7,7 @@ let availabilityDefinition = PackageDescription.SwiftSetting.unsafeFlags([ "-Xfrontend", "-define-availability", "-Xfrontend", - "SwiftStdlib 5.7:macOS 9999, iOS 9999, watchOS 9999, tvOS 9999", + "SwiftStdlib 5.7:macOS 13.0, iOS 16.0, watchOS 9.0, tvOS 16.0", "-Xfrontend", "-define-availability", "-Xfrontend", From 8e2f6091ad0bcf52d985b3afae2e9590f118ad11 Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Thu, 30 Mar 2023 15:51:32 -0600 Subject: [PATCH 02/10] short-circuit Character.isASCII checks --- Sources/_StringProcessing/Engine/MEBuiltins.swift | 4 ++-- Sources/_StringProcessing/_CharacterClassModel.swift | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Sources/_StringProcessing/Engine/MEBuiltins.swift b/Sources/_StringProcessing/Engine/MEBuiltins.swift index 36a6043fe..6f99058fd 100644 --- a/Sources/_StringProcessing/Engine/MEBuiltins.swift +++ b/Sources/_StringProcessing/Engine/MEBuiltins.swift @@ -38,9 +38,9 @@ extension Processor { return nil } - let asciiCheck = (char.isASCII && !isScalarSemantics) + let asciiCheck = !isStrictASCII || (scalar.isASCII && isScalarSemantics) - || !isStrictASCII + || char.isASCII var matched: Bool var next: Input.Index diff --git a/Sources/_StringProcessing/_CharacterClassModel.swift b/Sources/_StringProcessing/_CharacterClassModel.swift index c5f1f8ecd..548a9eea0 100644 --- a/Sources/_StringProcessing/_CharacterClassModel.swift +++ b/Sources/_StringProcessing/_CharacterClassModel.swift @@ -81,9 +81,10 @@ struct _CharacterClassModel: Hashable { let char = input[currentPosition] let scalar = input.unicodeScalars[currentPosition] let isScalarSemantics = matchLevel == .unicodeScalar - let asciiCheck = (char.isASCII && !isScalarSemantics) + + let asciiCheck = !isStrictASCII || (scalar.isASCII && isScalarSemantics) - || !isStrictASCII + || char.isASCII var matched: Bool var next: String.Index From 8ac83132c7f776a23c5d4a663466b4bdb4878748 Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Sun, 2 Apr 2023 16:07:33 -0600 Subject: [PATCH 03/10] Make benchmark try a few more times before giving up --- Sources/RegexBenchmark/BenchmarkRunner.swift | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Sources/RegexBenchmark/BenchmarkRunner.swift b/Sources/RegexBenchmark/BenchmarkRunner.swift index 1a62858c1..fb0fd3e79 100644 --- a/Sources/RegexBenchmark/BenchmarkRunner.swift +++ b/Sources/RegexBenchmark/BenchmarkRunner.swift @@ -1,6 +1,9 @@ import Foundation @_spi(RegexBenchmark) import _StringProcessing +/// The number of times to re-run the benchmark if results are too variang +private var rerunCount: Int { 3 } + struct BenchmarkRunner { let suiteName: String var suite: [any RegexBenchmark] = [] @@ -82,11 +85,16 @@ struct BenchmarkRunner { for b in suite { var result = measure(benchmark: b, samples: samples) if result.runtimeIsTooVariant { - print("Warning: Standard deviation > \(Stats.maxAllowedStdev*100)% for \(b.name)") - print(result.runtime) - print("Rerunning \(b.name)") - result = measure(benchmark: b, samples: result.runtime.samples*2) - print(result.runtime) + for _ in 0.. \(Stats.maxAllowedStdev*100)% for \(b.name)") + print(result.runtime) + print("Rerunning \(b.name)") + result = measure(benchmark: b, samples: result.runtime.samples*2) + print(result.runtime) + if !result.runtimeIsTooVariant { + break + } + } if result.runtimeIsTooVariant { fatalError("Benchmark \(b.name) is too variant") } From fe2795dc8146c00786c74d435c48fd9de4484354 Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Fri, 31 Mar 2023 09:29:01 -0600 Subject: [PATCH 04/10] wip: General ASCII fast-paths for builtin character classes --- .../_StringProcessing/Engine/MEBuiltins.swift | 8 ++ Sources/_StringProcessing/StringExtras.swift | 130 ++++++++++++++++++ .../_CharacterClassModel.swift | 11 +- 3 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 Sources/_StringProcessing/StringExtras.swift diff --git a/Sources/_StringProcessing/Engine/MEBuiltins.swift b/Sources/_StringProcessing/Engine/MEBuiltins.swift index 6f99058fd..4bebc6b0d 100644 --- a/Sources/_StringProcessing/Engine/MEBuiltins.swift +++ b/Sources/_StringProcessing/Engine/MEBuiltins.swift @@ -34,6 +34,14 @@ extension Processor { _ isStrictASCII: Bool, _ isScalarSemantics: Bool ) -> Input.Index? { + + // ASCII fast-path + if let (next, result) = input._quickMatch( + cc, at: currentPosition, isScalarSemantics: isScalarSemantics + ) { + return result == isInverted ? nil : next + } + guard let char = load(), let scalar = loadScalar() else { return nil } diff --git a/Sources/_StringProcessing/StringExtras.swift b/Sources/_StringProcessing/StringExtras.swift new file mode 100644 index 000000000..d2837ea51 --- /dev/null +++ b/Sources/_StringProcessing/StringExtras.swift @@ -0,0 +1,130 @@ + +/// TODO: better description +/// Whether this is the starting byte of a sub-300 (i.e. pre-combining scalar) scalars +private func _isSub300StartingByte(_ x: UInt8) -> Bool { + x < 0xCC +} +private func _isASCII(_ x: UInt8) -> Bool { + x < 0x80 +} + +private var _lineFeed: UInt8 { 0x0A } +private var _carriageReturn: UInt8 { 0x0D } +private var _lineTab: UInt8 { 0x0B } +private var _formFeed: UInt8 { 0x0C } +private var _space: UInt8 { 0x20 } +private var _tab: UInt8 { 0x09 } + + + +private var _0: UInt8 { 0x30 } +private var _9: UInt8 { 0x39 } +private func _isASCIINumber(_ x: UInt8) -> Bool { + return (_0..._9).contains(x) +} + +private var _a: UInt8 { 0x61 } +private var _z: UInt8 { 0x7A } +private var _A: UInt8 { 0x41 } +private var _Z: UInt8 { 0x5A } +private func _isASCIILetter(_ x: UInt8) -> Bool { + return (_a..._z).contains(x) || (_A..._Z).contains(x) +} + +private var _underscore: UInt8 { 0x5F } + + +extension String { + + + /// TODO: detailed description of nuanced semantics + func _asciiCharacter( + at idx: Index + ) -> (first: UInt8, next: Index, crLF: Bool)? { + // TODO: fastUTF8 version + + if idx == endIndex { + return nil + } + let base = utf8[idx] + guard _isASCII(base) else { return nil } + + var next = utf8.index(after: idx) + if next == utf8.endIndex { + return (first: base, next: next, crLF: false) + } + + let tail = utf8[next] + guard _isSub300StartingByte(tail) else { return nil } + + // Handle CR-LF: + if base == _carriageReturn && tail == _lineFeed { + utf8.formIndex(after: &next) + guard next == endIndex || _isSub300StartingByte(utf8[next]) else { + return nil + } + return (first: base, next: next, crLF: true) + } + + return (first: base, next: next, crLF: false) + } + + func _quickMatch( + _ cc: _CharacterClassModel.Representation, + at idx: Index, + isScalarSemantics: Bool + ) -> (next: Index, matchResult: Bool)? { + /// ASCII fast-paths + guard let (asciiValue, next, isCRLF) = _asciiCharacter( + at: idx + ) else { + return nil + } + + // TODO: bitvectors + switch cc { + case .any, .anyGrapheme, .anyScalar: + // TODO: should any scalar not consume CR-LF in scalar semantic mode? + return (next, true) + + case .digit: + if _isASCIINumber(asciiValue) { + return (next, true) + } + return (next, false) + + case .horizontalWhitespace: + switch asciiValue { + case _space, _tab: return (next, true) + default: return (next, false) + } + + case .verticalWhitespace, .newlineSequence: + switch asciiValue { + case _lineFeed, _carriageReturn, _lineTab, _formFeed: + // Scalar semantics: For `\v`, only advance past the CR instead of CR-LF + if isScalarSemantics && isCRLF && cc == .verticalWhitespace { + return (utf8.index(before: next), true) + } + return (next, true) + + default: + return (next, false) + } + + case .whitespace: + switch asciiValue { + case _space, _tab, _lineFeed, _lineTab, _formFeed, _carriageReturn: + return (next, true) + default: + return (next, false) + } + + case .word: + let matches = _isASCIINumber(asciiValue) || _isASCIILetter(asciiValue) || asciiValue == _underscore + return (next, matches) + } + } + +} + diff --git a/Sources/_StringProcessing/_CharacterClassModel.swift b/Sources/_StringProcessing/_CharacterClassModel.swift index 548a9eea0..d9a32a8e4 100644 --- a/Sources/_StringProcessing/_CharacterClassModel.swift +++ b/Sources/_StringProcessing/_CharacterClassModel.swift @@ -78,9 +78,18 @@ struct _CharacterClassModel: Hashable { guard currentPosition != input.endIndex else { return nil } + + let isScalarSemantics = matchLevel == .unicodeScalar + + // ASCII fast-path + if let (next, result) = input._quickMatch( + cc, at: currentPosition, isScalarSemantics: isScalarSemantics + ) { + return result == isInverted ? nil : next + } + let char = input[currentPosition] let scalar = input.unicodeScalars[currentPosition] - let isScalarSemantics = matchLevel == .unicodeScalar let asciiCheck = !isStrictASCII || (scalar.isASCII && isScalarSemantics) From e43c2e3841f942fbfd54b64b2303dc426f3f2404 Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Sat, 1 Apr 2023 10:33:24 -0600 Subject: [PATCH 05/10] WIP: sink some metrics --- Sources/_StringProcessing/Engine/Metrics.swift | 8 ++++++-- Sources/_StringProcessing/Engine/Processor.swift | 15 +++++++-------- Sources/_StringProcessing/Engine/Tracing.swift | 5 +++++ Sources/_StringProcessing/Utility/Traced.swift | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/Sources/_StringProcessing/Engine/Metrics.swift b/Sources/_StringProcessing/Engine/Metrics.swift index 753c3c3d1..013f63887 100644 --- a/Sources/_StringProcessing/Engine/Metrics.swift +++ b/Sources/_StringProcessing/Engine/Metrics.swift @@ -3,11 +3,15 @@ extension Processor { var instructionCounts: [Instruction.OpCode: Int] = [:] var backtracks: Int = 0 var resets: Int = 0 + var cycleCount: Int = 0 + + var isTracingEnabled: Bool = false + var shouldMeasureMetrics: Bool = false } func printMetrics() { print("===") - print("Total cycle count: \(cycleCount)") + print("Total cycle count: \(metrics.cycleCount)") print("Backtracks: \(metrics.backtracks)") print("Resets: \(metrics.resets)") print("Instructions:") @@ -30,7 +34,7 @@ extension Processor { } mutating func measureMetrics() { - if shouldMeasureMetrics { + if metrics.shouldMeasureMetrics { measure() } } diff --git a/Sources/_StringProcessing/Engine/Processor.swift b/Sources/_StringProcessing/Engine/Processor.swift index 18e355fb5..791f95b7b 100644 --- a/Sources/_StringProcessing/Engine/Processor.swift +++ b/Sources/_StringProcessing/Engine/Processor.swift @@ -86,10 +86,6 @@ struct Processor { var failureReason: Error? = nil - // MARK: Metrics, debugging, etc. - var cycleCount = 0 - var isTracingEnabled: Bool - let shouldMeasureMetrics: Bool var metrics: ProcessorMetrics = ProcessorMetrics() } @@ -116,8 +112,11 @@ extension Processor { self.subjectBounds = subjectBounds self.searchBounds = searchBounds self.matchMode = matchMode - self.isTracingEnabled = isTracingEnabled - self.shouldMeasureMetrics = shouldMeasureMetrics + + self.metrics = ProcessorMetrics( + isTracingEnabled: isTracingEnabled, + shouldMeasureMetrics: shouldMeasureMetrics) + self.currentPosition = searchBounds.lowerBound // Initialize registers with end of search bounds @@ -145,7 +144,7 @@ extension Processor { self.state = .inProgress self.failureReason = nil - if shouldMeasureMetrics { metrics.resets += 1 } + if metrics.shouldMeasureMetrics { metrics.resets += 1 } _checkInvariants() } @@ -402,7 +401,7 @@ extension Processor { registers.ints = intRegisters registers.positions = posRegisters - if shouldMeasureMetrics { metrics.backtracks += 1 } + if metrics.shouldMeasureMetrics { metrics.backtracks += 1 } } mutating func abort(_ e: Error? = nil) { diff --git a/Sources/_StringProcessing/Engine/Tracing.swift b/Sources/_StringProcessing/Engine/Tracing.swift index 725319b00..b0ce67555 100644 --- a/Sources/_StringProcessing/Engine/Tracing.swift +++ b/Sources/_StringProcessing/Engine/Tracing.swift @@ -9,7 +9,12 @@ // //===----------------------------------------------------------------------===// + +// TODO: Remove this protocol (and/or reuse it for something like a FastProcessor) extension Processor: TracedProcessor { + var cycleCount: Int { metrics.cycleCount } + var isTracingEnabled: Bool { metrics.isTracingEnabled } + var isFailState: Bool { state == .fail } var isAcceptState: Bool { state == .accept } diff --git a/Sources/_StringProcessing/Utility/Traced.swift b/Sources/_StringProcessing/Utility/Traced.swift index 112a601b1..198564fe1 100644 --- a/Sources/_StringProcessing/Utility/Traced.swift +++ b/Sources/_StringProcessing/Utility/Traced.swift @@ -13,7 +13,7 @@ // TODO: Place shared formatting and trace infrastructure here protocol Traced { - var isTracingEnabled: Bool { get set } + var isTracingEnabled: Bool { get } } protocol TracedProcessor: ProcessorProtocol, Traced { From 2ac070a1d5103525fa6c814027ed9e0b9a7a29cb Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Sat, 1 Apr 2023 10:52:23 -0600 Subject: [PATCH 06/10] wip: refactor metrics out, and void their storage --- .../_StringProcessing/Engine/Metrics.swift | 57 ++++++++++++++++++- .../_StringProcessing/Engine/Processor.swift | 24 +++----- Sources/_StringProcessing/Executor.swift | 4 +- 3 files changed, 65 insertions(+), 20 deletions(-) diff --git a/Sources/_StringProcessing/Engine/Metrics.swift b/Sources/_StringProcessing/Engine/Metrics.swift index 013f63887..dfde6c0d0 100644 --- a/Sources/_StringProcessing/Engine/Metrics.swift +++ b/Sources/_StringProcessing/Engine/Metrics.swift @@ -1,4 +1,5 @@ extension Processor { +#if PROCESSOR_MEASUREMENTS_ENABLED struct ProcessorMetrics { var instructionCounts: [Instruction.OpCode: Int] = [:] var backtracks: Int = 0 @@ -7,8 +8,61 @@ extension Processor { var isTracingEnabled: Bool = false var shouldMeasureMetrics: Bool = false + + init(isTracingEnabled: Bool, shouldMeasureMetrics: Bool) { + self.isTracingEnabled = isTracingEnabled + self.shouldMeasureMetrics = shouldMeasureMetrics + } } - +#else + struct ProcessorMetrics { + var isTracingEnabled: Bool { false } + var shouldMeasureMetrics: Bool { false } + var cycleCount: Int { 0 } + + init(isTracingEnabled: Bool, shouldMeasureMetrics: Bool) { } + } +#endif +} + +extension Processor { + + mutating func startCycleMetrics() { +#if PROCESSOR_MEASUREMENTS_ENABLED + if metrics.cycleCount == 0 { + trace() + measureMetrics() + } +#endif + } + + mutating func endCycleMetrics() { +#if PROCESSOR_MEASUREMENTS_ENABLED + metrics.cycleCount += 1 + trace() + measureMetrics() + _checkInvariants() +#endif + } +} + +extension Processor.ProcessorMetrics { + + mutating func addReset() { +#if PROCESSOR_MEASUREMENTS_ENABLED + self.resets += 1 +#endif + } + + mutating func addBacktrack() { +#if PROCESSOR_MEASUREMENTS_ENABLED + self.backtracks += 1 +#endif + } +} + +extension Processor { +#if PROCESSOR_MEASUREMENTS_ENABLED func printMetrics() { print("===") print("Total cycle count: \(metrics.cycleCount)") @@ -38,4 +92,5 @@ extension Processor { measure() } } +#endif } diff --git a/Sources/_StringProcessing/Engine/Processor.swift b/Sources/_StringProcessing/Engine/Processor.swift index 791f95b7b..c90b0fd4c 100644 --- a/Sources/_StringProcessing/Engine/Processor.swift +++ b/Sources/_StringProcessing/Engine/Processor.swift @@ -86,7 +86,7 @@ struct Processor { var failureReason: Error? = nil - var metrics: ProcessorMetrics = ProcessorMetrics() + var metrics: ProcessorMetrics } extension Processor { @@ -143,8 +143,8 @@ extension Processor { self.state = .inProgress self.failureReason = nil - - if metrics.shouldMeasureMetrics { metrics.resets += 1 } + + metrics.addReset() _checkInvariants() } @@ -400,8 +400,8 @@ extension Processor { storedCaptures = capEnds registers.ints = intRegisters registers.positions = posRegisters - - if metrics.shouldMeasureMetrics { metrics.backtracks += 1 } + + metrics.addBacktrack() } mutating func abort(_ e: Error? = nil) { @@ -440,18 +440,8 @@ extension Processor { _checkInvariants() assert(state == .inProgress) -#if PROCESSOR_MEASUREMENTS_ENABLED - if cycleCount == 0 { - trace() - measureMetrics() - } - defer { - cycleCount += 1 - trace() - measureMetrics() - _checkInvariants() - } -#endif + startCycleMetrics() + defer { endCycleMetrics() } let (opcode, payload) = fetch().destructure switch opcode { diff --git a/Sources/_StringProcessing/Executor.swift b/Sources/_StringProcessing/Executor.swift index 253858d1f..0453fcd80 100644 --- a/Sources/_StringProcessing/Executor.swift +++ b/Sources/_StringProcessing/Executor.swift @@ -31,7 +31,7 @@ struct Executor { subjectBounds: subjectBounds, searchBounds: searchBounds) #if PROCESSOR_MEASUREMENTS_ENABLED - defer { if cpu.shouldMeasureMetrics { cpu.printMetrics() } } + defer { if cpu.metrics.shouldMeasureMetrics { cpu.printMetrics() } } #endif var low = searchBounds.lowerBound let high = searchBounds.upperBound @@ -60,7 +60,7 @@ struct Executor { var cpu = engine.makeProcessor( input: input, bounds: subjectBounds, matchMode: mode) #if PROCESSOR_MEASUREMENTS_ENABLED - defer { if cpu.shouldMeasureMetrics { cpu.printMetrics() } } + defer { if cpu.metrics.shouldMeasureMetrics { cpu.printMetrics() } } #endif return try _match(input, from: subjectBounds.lowerBound, using: &cpu) } From d5db3a630ef38976b854dd35b17e3dac75de7e64 Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Sat, 1 Apr 2023 10:23:37 -0600 Subject: [PATCH 07/10] wip: string internals --- .../_StringProcessing/Engine/Processor.swift | 7 + Sources/_StringProcessing/StringExtras.swift | 180 +++++++++++++++++- 2 files changed, 185 insertions(+), 2 deletions(-) diff --git a/Sources/_StringProcessing/Engine/Processor.swift b/Sources/_StringProcessing/Engine/Processor.swift index c90b0fd4c..dd6ddf783 100644 --- a/Sources/_StringProcessing/Engine/Processor.swift +++ b/Sources/_StringProcessing/Engine/Processor.swift @@ -87,6 +87,9 @@ struct Processor { var failureReason: Error? = nil var metrics: ProcessorMetrics + + /// Set if the string has fast contiguous UTF-8 available + let fastUTF8: UnsafeRawBufferPointer? = nil } extension Processor { @@ -124,6 +127,8 @@ extension Processor { self.storedCaptures = Array( repeating: .init(), count: program.registerInfo.captures) + // print(MemoryLayout.size) + _checkInvariants() } @@ -264,6 +269,8 @@ extension Processor { return true } + @inline(never) + @_effects(releasenone) func loadScalar() -> Unicode.Scalar? { currentPosition < end ? input.unicodeScalars[currentPosition] : nil } diff --git a/Sources/_StringProcessing/StringExtras.swift b/Sources/_StringProcessing/StringExtras.swift index d2837ea51..35c12c407 100644 --- a/Sources/_StringProcessing/StringExtras.swift +++ b/Sources/_StringProcessing/StringExtras.swift @@ -34,9 +34,8 @@ private func _isASCIILetter(_ x: UInt8) -> Bool { private var _underscore: UInt8 { 0x5F } -extension String { - +extension String { /// TODO: detailed description of nuanced semantics func _asciiCharacter( at idx: Index @@ -128,3 +127,180 @@ extension String { } +/// MARK: TODO: Better as SPI or new low-level interfaces... + +extension String { + var _object: _StringObject { + unsafeBitCast(self, to: _StringObject.self) + } + + var _unsafeFastUTF8: UnsafeRawBufferPointer? { + // TODO: platform support, or a stdlib alternative +#if arch(i386) || arch(arm) || arch(arm64_32) || arch(wasm32) + return nil +#endif + return _object.fastUTF8IfAvailable + } + +} + +internal struct _StringObject { + // Abstract the count and performance-flags containing word + struct CountAndFlags { + var _storage: UInt64 + + @inline(__always) + internal init(zero: ()) { self._storage = 0 } + } + + internal var _countAndFlagsBits: UInt64 + + /// Bastardization, since we can't access Builtin.BridgeObject + internal var discriminatedObjectRawBits: UInt64 + + @inline(__always) + internal var _countAndFlags: CountAndFlags { + _internalInvariant(!isSmall) + return CountAndFlags(rawUnchecked: _countAndFlagsBits) + } + + // Whether this string is native, i.e. tail-allocated and nul-terminated, + // presupposing it is both large and fast + @inline(__always) + internal var largeFastIsTailAllocated: Bool { + _internalInvariant(isLarge && providesFastUTF8) + return _countAndFlags.isTailAllocated + } + + // @inlinable @_transparent + // internal var discriminatedObjectRawBits: UInt64 { + //#if arch(i386) || arch(arm) || arch(arm64_32) || arch(wasm32) + // let low32: UInt + // switch _variant { + // case .immortal(let bitPattern): + // low32 = bitPattern + // case .native(let storage): + // low32 = Builtin.reinterpretCast(storage) + // case .bridged(let object): + // low32 = Builtin.reinterpretCast(object) + // } + // + // return UInt64(truncatingIfNeeded: _discriminator) &<< 56 + // | UInt64(truncatingIfNeeded: low32) + //#else + // return unsafeBitCast(_object) + //#endif + // } + + @inline(__always) + internal var isSmall: Bool { +#if os(Android) && arch(arm64) + return (discriminatedObjectRawBits & 0x0020_0000_0000_0000) != 0 +#else + return (discriminatedObjectRawBits & 0x2000_0000_0000_0000) != 0 +#endif + } + + @inline(__always) + internal var isLarge: Bool { return !isSmall } + + // Whether this string can provide access to contiguous UTF-8 code units: + // - Small strings can by spilling to the stack + // - Large native strings can through an offset + // - Shared strings can: + // - Cocoa strings which respond to e.g. CFStringGetCStringPtr() + // - Non-Cocoa shared strings + @inline(__always) + internal var providesFastUTF8: Bool { +#if os(Android) && arch(arm64) + return (discriminatedObjectRawBits & 0x0010_0000_0000_0000) == 0 +#else + return (discriminatedObjectRawBits & 0x1000_0000_0000_0000) == 0 +#endif + } + + /// A bastardization of fastUTF8 from StringObject.swift. For now, + /// exclude shared strings. + @inline(__always) + var fastUTF8IfAvailable: UnsafeRawBufferPointer? { + guard self.largeFastIsTailAllocated else { + return nil + } + return UnsafeRawBufferPointer( + start: self.nativeUTF8Start, count: self.largeCount) + } + + @inline(__always) + internal var largeCount: Int { + _internalInvariant(isLarge) + return _countAndFlags.count + } + + @inline(__always) + internal var nativeUTF8Start: UnsafePointer { + _internalInvariant(largeFastIsTailAllocated) + return UnsafePointer( + bitPattern: largeAddressBits &+ _StringObject.nativeBias + )._unsafelyUnwrappedUnchecked + } + + + @inline(__always) + internal var largeAddressBits: UInt { + _internalInvariant(isLarge) + return UInt(truncatingIfNeeded: + discriminatedObjectRawBits & Nibbles.largeAddressMask) + } + + enum Nibbles {} + + @inline(__always) + internal static var nativeBias: UInt { +#if arch(i386) || arch(arm) || arch(arm64_32) || arch(wasm32) + return 20 +#else + return 32 +#endif + } +} + +extension _StringObject.Nibbles { + // Mask for address bits, i.e. non-discriminator and non-extra high bits + @inline(__always) + static internal var largeAddressMask: UInt64 { +#if os(Android) && arch(arm64) + return 0xFF0F_FFFF_FFFF_FFFF +#else + return 0x0FFF_FFFF_FFFF_FFFF +#endif + } + +} + +extension _StringObject.CountAndFlags { + internal typealias RawBitPattern = UInt64 + + @inline(__always) + internal init(rawUnchecked bits: RawBitPattern) { + self._storage = bits + } + + @inline(__always) + internal static var isTailAllocatedMask: UInt64 { + 0x1000_0000_0000_0000 + } + + @inline(__always) + internal static var countMask: UInt64 { 0x0000_FFFF_FFFF_FFFF } + + @inline(__always) + internal var count: Int { + return Int( + truncatingIfNeeded: _storage & _StringObject.CountAndFlags.countMask) + } + + @inline(__always) + internal var isTailAllocated: Bool { + return 0 != _storage & _StringObject.CountAndFlags.isTailAllocatedMask + } +} From 1c19f9087a7a8643b9a67aa428b9b379d24943d6 Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Sat, 1 Apr 2023 11:10:33 -0600 Subject: [PATCH 08/10] wip: set fastUTF8 --- Sources/_StringProcessing/Engine/Processor.swift | 13 ++++++++++++- Sources/_StringProcessing/StringExtras.swift | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Sources/_StringProcessing/Engine/Processor.swift b/Sources/_StringProcessing/Engine/Processor.swift index dd6ddf783..09d191f36 100644 --- a/Sources/_StringProcessing/Engine/Processor.swift +++ b/Sources/_StringProcessing/Engine/Processor.swift @@ -89,7 +89,7 @@ struct Processor { var metrics: ProcessorMetrics /// Set if the string has fast contiguous UTF-8 available - let fastUTF8: UnsafeRawBufferPointer? = nil + let fastUTF8: UnsafeRawBufferPointer? } extension Processor { @@ -128,6 +128,7 @@ extension Processor { repeating: .init(), count: program.registerInfo.captures) // print(MemoryLayout.size) + self.fastUTF8 = input._unsafeFastUTF8 _checkInvariants() } @@ -160,6 +161,16 @@ extension Processor { assert(subjectBounds.upperBound <= input.endIndex) assert(currentPosition >= searchBounds.lowerBound) assert(currentPosition <= searchBounds.upperBound) + + assert({ + guard let utf8 = self.fastUTF8 else { return true } + var copy = input + return copy.withUTF8 { + let base = UnsafeRawPointer($0.baseAddress!) + return utf8.baseAddress == base + } + }()) + } } diff --git a/Sources/_StringProcessing/StringExtras.swift b/Sources/_StringProcessing/StringExtras.swift index 35c12c407..5af453791 100644 --- a/Sources/_StringProcessing/StringExtras.swift +++ b/Sources/_StringProcessing/StringExtras.swift @@ -223,7 +223,7 @@ internal struct _StringObject { /// exclude shared strings. @inline(__always) var fastUTF8IfAvailable: UnsafeRawBufferPointer? { - guard self.largeFastIsTailAllocated else { + guard self.isLarge && self.providesFastUTF8 && self.largeFastIsTailAllocated else { return nil } return UnsafeRawBufferPointer( From 9827389ba51398dc440662b5ce33be6681a0d82b Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Sun, 2 Apr 2023 14:45:07 -0600 Subject: [PATCH 09/10] wip: more of the fastUTF8 fast path work --- .../_StringProcessing/Engine/Processor.swift | 54 ++++++++++++++----- Sources/_StringProcessing/StringExtras.swift | 6 +++ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/Sources/_StringProcessing/Engine/Processor.swift b/Sources/_StringProcessing/Engine/Processor.swift index 09d191f36..cf3aa6293 100644 --- a/Sources/_StringProcessing/Engine/Processor.swift +++ b/Sources/_StringProcessing/Engine/Processor.swift @@ -35,7 +35,7 @@ struct Processor { /// of the search. `input` can be a "supersequence" of the subject, while /// `input[subjectBounds]` is the logical entity that is being searched. let input: Input - + /// The bounds of the logical subject in `input`. /// /// `subjectBounds` represents the bounds of the string or substring that a @@ -46,7 +46,7 @@ struct Processor { /// `subjectBounds` is always equal to or a subrange of /// `input.startIndex.. - + /// The bounds within the subject for an individual search. /// /// `searchBounds` is equal to `subjectBounds` in some cases, but can be a @@ -62,7 +62,7 @@ struct Processor { let instructions: InstructionList // MARK: Resettable state - + /// The current search position while processing. /// /// `currentPosition` must always be in the range `subjectBounds` or equal @@ -81,7 +81,7 @@ struct Processor { var wordIndexCache: Set? = nil var wordIndexMaxIndex: String.Index? = nil - + var state: State = .inProgress var failureReason: Error? = nil @@ -89,7 +89,7 @@ struct Processor { var metrics: ProcessorMetrics /// Set if the string has fast contiguous UTF-8 available - let fastUTF8: UnsafeRawBufferPointer? + let fastUTF8: UnsafeRawPointer? } extension Processor { @@ -128,7 +128,7 @@ extension Processor { repeating: .init(), count: program.registerInfo.captures) // print(MemoryLayout.size) - self.fastUTF8 = input._unsafeFastUTF8 + self.fastUTF8 = input._unsafeFastUTF8?.baseAddress _checkInvariants() } @@ -167,7 +167,7 @@ extension Processor { var copy = input return copy.withUTF8 { let base = UnsafeRawPointer($0.baseAddress!) - return utf8.baseAddress == base + return utf8 == base } }()) @@ -201,7 +201,7 @@ extension Processor { currentPosition = idx return true } - + // Advances in unicode scalar view mutating func consumeScalar(_ n: Distance) -> Bool { guard let idx = input.unicodeScalars.index( @@ -283,10 +283,38 @@ extension Processor { @inline(never) @_effects(releasenone) func loadScalar() -> Unicode.Scalar? { - currentPosition < end ? input.unicodeScalars[currentPosition] : nil + guard currentPosition < end else { return nil } +// if let utf8 = self.fastUTF8 { +// let firstByte = utf8[currentPosition.encodedOffset] +// if firstByte < 0x80 { +// let returnValue = Unicode.Scalar(firstByte) +// // TODO: More comprehensive assertion framework to test before and after +// // TODO: unsafe-ish optimizations +// assert(returnValue == input.unicodeScalars[currentPosition]) +// +// return returnValue +// } +// +// } + return input.unicodeScalars[currentPosition] } - + func _doMatchScalar(_ s: Unicode.Scalar, _ boundaryCheck: Bool) -> Input.Index? { + guard currentPosition < end else { return nil } + + if s.isASCII, let utf8 = self.fastUTF8 { + let nextByteIdx = input.utf8.index(after: currentPosition) + if utf8.loadByte(currentPosition) == s.value { + // TODO: comprehensive assertion framework + assert(s == input.unicodeScalars[currentPosition]) + if (!boundaryCheck || input.isOnGraphemeClusterBoundary(nextByteIdx)) { + return nextByteIdx + } + } + return nil + } // 13-22ms, after: 22-25ms ??? + // Now down to 3ms + if s == loadScalar(), let idx = input.unicodeScalars.index( currentPosition, @@ -298,7 +326,7 @@ extension Processor { return nil } } - + mutating func matchScalar(_ s: Unicode.Scalar, boundaryCheck: Bool) -> Bool { guard let next = _doMatchScalar(s, boundaryCheck) else { signalFailure() @@ -372,7 +400,7 @@ extension Processor { _uncheckedForcedConsumeOne() return true } - + // Matches the next scalar if it is not a newline mutating func matchAnyNonNewlineScalar() -> Bool { guard let s = loadScalar(), !s.isNewline else { @@ -453,7 +481,7 @@ extension Processor { // TODO: What should we do here? fatalError("Invalid code: Tried to clear save points when empty") } - + mutating func cycle() { _checkInvariants() assert(state == .inProgress) diff --git a/Sources/_StringProcessing/StringExtras.swift b/Sources/_StringProcessing/StringExtras.swift index 5af453791..20d2ac5f1 100644 --- a/Sources/_StringProcessing/StringExtras.swift +++ b/Sources/_StringProcessing/StringExtras.swift @@ -304,3 +304,9 @@ extension _StringObject.CountAndFlags { return 0 != _storage & _StringObject.CountAndFlags.isTailAllocatedMask } } + +extension UnsafeRawPointer { + func loadByte(_ offset: String.Index) -> UInt8 { + self.load(fromByteOffset: offset.encodedOffset, as: UInt8.self) + } +} From fe7352b4f1333bf2f2c64f50cca04963018cf1e4 Mon Sep 17 00:00:00 2001 From: Michael Ilseman Date: Mon, 3 Apr 2023 11:31:58 -0600 Subject: [PATCH 10/10] wip --- Sources/RegexBenchmark/BenchmarkRunner.swift | 2 +- Sources/_StringProcessing/StringExtras.swift | 32 +++++--------------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/Sources/RegexBenchmark/BenchmarkRunner.swift b/Sources/RegexBenchmark/BenchmarkRunner.swift index fb0fd3e79..87fc6c186 100644 --- a/Sources/RegexBenchmark/BenchmarkRunner.swift +++ b/Sources/RegexBenchmark/BenchmarkRunner.swift @@ -89,7 +89,7 @@ struct BenchmarkRunner { print("Warning: Standard deviation > \(Stats.maxAllowedStdev*100)% for \(b.name)") print(result.runtime) print("Rerunning \(b.name)") - result = measure(benchmark: b, samples: result.runtime.samples*2) + result = measure(benchmark: b, samples: result.runtime.samples) print(result.runtime) if !result.runtimeIsTooVariant { break diff --git a/Sources/_StringProcessing/StringExtras.swift b/Sources/_StringProcessing/StringExtras.swift index 20d2ac5f1..ce177b548 100644 --- a/Sources/_StringProcessing/StringExtras.swift +++ b/Sources/_StringProcessing/StringExtras.swift @@ -139,7 +139,8 @@ extension String { #if arch(i386) || arch(arm) || arch(arm64_32) || arch(wasm32) return nil #endif - return _object.fastUTF8IfAvailable + // TODO: Add in shared string support + return _object.nativeUTF8IfAvailable } } @@ -155,7 +156,8 @@ internal struct _StringObject { internal var _countAndFlagsBits: UInt64 - /// Bastardization, since we can't access Builtin.BridgeObject + // NOTE: We store the raw bits instead of BridgeObject, because we don't + // have access to bridge object internal var discriminatedObjectRawBits: UInt64 @inline(__always) @@ -172,26 +174,6 @@ internal struct _StringObject { return _countAndFlags.isTailAllocated } - // @inlinable @_transparent - // internal var discriminatedObjectRawBits: UInt64 { - //#if arch(i386) || arch(arm) || arch(arm64_32) || arch(wasm32) - // let low32: UInt - // switch _variant { - // case .immortal(let bitPattern): - // low32 = bitPattern - // case .native(let storage): - // low32 = Builtin.reinterpretCast(storage) - // case .bridged(let object): - // low32 = Builtin.reinterpretCast(object) - // } - // - // return UInt64(truncatingIfNeeded: _discriminator) &<< 56 - // | UInt64(truncatingIfNeeded: low32) - //#else - // return unsafeBitCast(_object) - //#endif - // } - @inline(__always) internal var isSmall: Bool { #if os(Android) && arch(arm64) @@ -219,10 +201,10 @@ internal struct _StringObject { #endif } - /// A bastardization of fastUTF8 from StringObject.swift. For now, - /// exclude shared strings. + /// A modification of fastUTF8 from StringObject.swift, as we don't have + /// access to shared string internals. @inline(__always) - var fastUTF8IfAvailable: UnsafeRawBufferPointer? { + var nativeUTF8IfAvailable: UnsafeRawBufferPointer? { guard self.isLarge && self.providesFastUTF8 && self.largeFastIsTailAllocated else { return nil }