Schema-based serialization working branch

.github/workflows/continuous-integration.yml

@@ -66,7 +66,6 @@ jobs:
       - name: Build & Run smithy-swift Kotlin Unit Tests
         run: ./gradlew build
       - name: Build & Run smithy-swift Swift Unit Tests
-        timeout-minutes: 15
         run: |
           set -o pipefail && \
           NSUnbufferedIO=YES xcodebuild \

Package.swift

@@ -30,6 +30,7 @@ let package = Package(
     ],
     products: [
         .library(name: "Smithy", targets: ["Smithy"]),
+        .library(name: "RPCv2CBOR", targets: ["RPCv2CBOR"]),
         .library(name: "SmithySerialization", targets: ["SmithySerialization"]),
         .library(name: "ClientRuntime", targets: ["ClientRuntime"]),
         .library(name: "SmithyRetriesAPI", targets: ["SmithyRetriesAPI"]),
@@ -286,8 +287,6 @@ let package = Package(
         .target(
             name: "SmithyCBOR",
             dependencies: [
-                "SmithyReadWrite",
-                "SmithyTimestamps",
                 "Smithy",
                 "SmithySerialization",
                 .product(name: "AwsCommonRuntimeKit", package: "aws-crt-swift")
@@ -318,6 +317,15 @@ let package = Package(
             ],
             resources: [ .process("Resources") ]
         ),
+        .target(
+            name: "RPCv2CBOR",
+            dependencies: [
+                "ClientRuntime",
+                "Smithy",
+                "SmithySerialization",
+                "SmithyCBOR",
+            ]
+        ),
         .testTarget(
             name: "ClientRuntimeTests",
             dependencies: [
@@ -410,5 +418,9 @@ let package = Package(
             dependencies: ["SmithyCodegenCore"],
             resources: [ .process("Resources") ]
         ),
+        .testTarget(
+            name: "SmithySerializationTests",
+            dependencies: ["SmithySerialization"]
+        ),
     ].compactMap { $0 }
 )

Plugins/SmithyCodeGeneratorPlugin/SmithyCodeGeneratorPlugin.swift

@@ -38,19 +38,22 @@ struct SmithyCodeGeneratorPlugin: BuildToolPlugin {
         in outputDirectoryPath: Path,
         with generatorToolPath: Path
     ) throws -> Command? {
-        // Skip any file that isn't the smithy-model-info.json for this service.
-        guard inputPath.lastComponent == "smithy-model-info.json" else { return nil }
+        // Skip any file that isn't the swift-settings.json for this service.
+        guard inputPath.lastComponent == "swift-settings.json" else { return nil }
 
         let currentWorkingDirectoryFileURL = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
 
         // Get the smithy-model-info.json file contents.
         let modelInfoData = try Data(contentsOf: URL(fileURLWithPath: inputPath.string))
         let smithyModelInfo = try JSONDecoder().decode(SmithyModelInfo.self, from: modelInfoData)
 
-        // Get the service ID & model path & settings sdkId from smithy-model-info.
+        // Get the fields from smithy-model-info.
         let service = smithyModelInfo.service
-        let modelPathURL = currentWorkingDirectoryFileURL.appendingPathComponent(smithyModelInfo.path)
+        let sdkId = smithyModelInfo.sdkId
+        let modelPathURL = currentWorkingDirectoryFileURL.appendingPathComponent(smithyModelInfo.modelPath)
         let modelPath = Path(modelPathURL.path)
+        let internalClient = smithyModelInfo.internalClient
+        let operations = smithyModelInfo.operations.joined(separator: ",")
 
         // Construct the Schemas.swift path.
         let schemasSwiftPath = outputDirectoryPath.appending("\(name)Schemas.swift")
@@ -61,32 +64,65 @@ struct SmithyCodeGeneratorPlugin: BuildToolPlugin {
         // Construct the Deserialize.swift path.
         let deserializeSwiftPath = outputDirectoryPath.appending("\(name)Deserialize.swift")
 
+        // Construct the Deserialize.swift path.
+        let typeRegistrySwiftPath = outputDirectoryPath.appending("\(name)TypeRegistry.swift")
+
+        // Construct the Operations.swift path.
+        let operationsSwiftPath = outputDirectoryPath.appending("\(name)Operations.swift")
+
+        var arguments: [any CustomStringConvertible] = [
+            service,
+            modelPath,
+            "--internal", "\(internalClient)",
+            "--sdk-id", sdkId,
+            "--schemas-path", schemasSwiftPath,
+            "--serialize-path", serializeSwiftPath,
+            "--deserialize-path", deserializeSwiftPath,
+            "--type-registry-path", typeRegistrySwiftPath,
+            "--operations-path", operationsSwiftPath,
+            "--schemas-path", schemasSwiftPath,
+            "--serialize-path", serializeSwiftPath,
+            "--deserialize-path", deserializeSwiftPath
+        ]
+
+        if !operations.isEmpty {
+            arguments.append(contentsOf: ["--operations", operations])
+        }
+
         // Construct the build command that invokes SmithyCodegenCLI.
         return .buildCommand(
-            displayName: "Generating Swift source files from model file \(smithyModelInfo.path)",
+            displayName: "Generating Swift source files from model file \(smithyModelInfo.modelPath)",
             executable: generatorToolPath,
-            arguments: [
-                service,
-                modelPath,
-                "--schemas-path", schemasSwiftPath,
-                "--serialize-path", serializeSwiftPath,
-                "--deserialize-path", deserializeSwiftPath
-            ],
+            arguments: arguments,
             inputFiles: [inputPath, modelPath],
             outputFiles: [
                 schemasSwiftPath,
                 serializeSwiftPath,
                 deserializeSwiftPath,
+                typeRegistrySwiftPath,
+                operationsSwiftPath,
             ]
         )
     }
 }
 
-/// Codable structure for reading the contents of `smithy-model-info.json`
+/// Decodable structure for reading the contents of `smithy-model-info.json`
 private struct SmithyModelInfo: Decodable {
     /// The shape ID of the service being generated.  Must exist in the model.
     let service: String
 
+    /// The name to be used for the enclosing module.
+    let module: String
+
+    /// The `sdkId` used by the Smithy-based code generator.
+    let sdkId: String
+
+    /// Set to `true` if the client should be rendered for internal use.
+    let internalClient: Bool
+
+    /// A list of operations to be included in the client.  If omitted or empty, all operations are included.
+    let operations: [String]
+
     /// The path to the model, from the root of the target's project.  Required.
-    let path: String
+    let modelPath: String
 }

Sources/ClientRuntime/Config/Context+SchemaBased.swift

@@ -0,0 +1,40 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import struct Smithy.AttributeKey
+import class Smithy.Context
+import class Smithy.ContextBuilder
+import protocol SmithySerialization.ClientProtocol
+import struct SmithySerialization.Operation
+import protocol SmithySerialization.OperationProperties
+
+extension Context {
+
+    func operation<Input, Output>() -> Operation<Input, Output>? {
+        get(key: operationContextKey) as? Operation<Input, Output>
+    }
+
+    var clientProtocol: (any ClientProtocol)? {
+        `get`(key: clientProtocolKey)
+    }
+}
+
+extension ContextBuilder {
+
+    func withOperation<Input, Output>(_ operation: SmithySerialization.Operation<Input, Output>) -> ContextBuilder {
+        attributes.set(key: operationContextKey, value: operation)
+        return self
+    }
+
+    func withClientProtocol(_ clientProtocol: any ClientProtocol) -> ContextBuilder {
+        attributes.set(key: clientProtocolKey, value: clientProtocol)
+        return self
+    }
+}
+
+private let operationContextKey = AttributeKey<any OperationProperties>(name: "OperationContextKey")
+private let clientProtocolKey = AttributeKey<any ClientProtocol>(name: "ClientProtocolKey")

Sources/ClientRuntime/Networking/Http/HTTPError.swift

@@ -12,5 +12,5 @@ import class SmithyHTTPAPI.HTTPResponse
 public protocol HTTPError {
 
     /// The HTTP/HTTPS response that resulted in this error.
-    var httpResponse: HTTPResponse { get }
+    var httpResponse: HTTPResponse { get set }
 }

Sources/ClientRuntime/Networking/Http/Middlewares/DeserializeMiddleware.swift

@@ -62,7 +62,7 @@ extension DeserializeMiddleware: ResponseMessageDeserializer {
 }
 
 // Calculates & returns estimated skew.
-func getEstimatedSkew(now: Date, responseDateString: String) -> TimeInterval {
+public func getEstimatedSkew(now: Date, responseDateString: String) -> TimeInterval {
     let dateFormatter = DateFormatter()
     dateFormatter.dateFormat = "EEE, dd MMM yyyy HH:mm:ss z"
     dateFormatter.locale = Locale(identifier: "en_US_POSIX")

Sources/ClientRuntime/Networking/Http/Middlewares/RequestBody/SchemaBodyMiddleware.swift

@@ -0,0 +1,44 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import struct Foundation.Data
+import Smithy
+import SmithyHTTPAPI
+import protocol SmithySerialization.ClientProtocol
+import protocol SmithySerialization.Codec
+import protocol SmithySerialization.DeserializableStruct
+import struct SmithySerialization.Operation
+import protocol SmithySerialization.SerializableStruct
+
+public struct SchemaBodyMiddleware<Input: SerializableStruct, Output: DeserializableStruct, CP: ClientProtocol> {
+    public let id: Swift.String = "BodyMiddleware"
+    let operation: Operation<Input, Output>
+    let clientProtocol: CP
+
+    public init(_ operation: Operation<Input, Output>, _ clientProtocol: CP) {
+        self.operation = operation
+        self.clientProtocol = clientProtocol
+    }
+}
+
+extension SchemaBodyMiddleware: RequestMessageSerializer {
+    public typealias InputType = Input
+    public typealias RequestType = CP.RequestType
+
+    public func apply(input: Input, builder: RequestType.RequestBuilderType, attributes: Context) throws {
+        do {
+            return try clientProtocol.serializeRequest(
+                operation: operation,
+                input: input,
+                requestBuilder: builder,
+                context: attributes
+            )
+        } catch {
+            throw ClientError.serializationFailed(error.localizedDescription)
+        }
+    }
+}

Sources/ClientRuntime/Networking/Http/Middlewares/SchemaDeserializeMiddleware.swift

@@ -0,0 +1,68 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import struct Foundation.Data
+import struct Foundation.Date
+import class Foundation.DateFormatter
+import struct Foundation.Locale
+import struct Foundation.TimeInterval
+import struct Foundation.TimeZone
+import struct Foundation.UUID
+import class Smithy.Context
+import protocol Smithy.ResponseMessageDeserializer
+import SmithyHTTPAPI
+import protocol SmithySerialization.ClientProtocol
+import protocol SmithySerialization.DeserializableStruct
+import struct SmithySerialization.Operation
+import protocol SmithySerialization.SerializableStruct
+
+public struct SchemaDeserializeMiddleware<
+    Input: SerializableStruct,
+    Output: DeserializableStruct,
+    ClientProtocol: SmithySerialization.ClientProtocol
+> where ClientProtocol.ResponseType == HTTPResponse {
+    public var id: String = "Deserialize"
+    let operation: Operation<Input, Output>
+    let clientProtocol: ClientProtocol
+
+    public init(_ operation: Operation<Input, Output>, _ clientProtocol: ClientProtocol) {
+        self.operation = operation
+        self.clientProtocol = clientProtocol
+    }
+}
+
+extension SchemaDeserializeMiddleware: ResponseMessageDeserializer {
+
+    public func deserialize(response: ClientProtocol.ResponseType, attributes: Context) async throws -> Output {
+        if let responseDateString = response.headers.value(for: "Date") {
+            let estimatedSkew = getEstimatedSkew(now: Date(), responseDateString: responseDateString)
+            attributes.estimatedSkew = estimatedSkew
+        }
+
+        // check if the response body was affected by a previous middleware
+        if let contextBody = attributes.httpResponse?.body {
+            response.body = contextBody
+        }
+
+        // if the response is an error and the response body is a stream, we need to cache the stream so
+        // that it can be read again.
+        //
+        // error deserialization reads the stream multiple times to first deserialize the protocol error
+        // eg. [RestJSONError](https://github.com/awslabs/aws-sdk-swift/blob/d1d18eefb7457ed27d416b372573a1f815004eb1/Sources/Core/AWSClientRuntime/Protocols/RestJSON/RestJSONError.swift#L38,
+        // and then the service error eg. [AccountNotFoundException](https://github.com/awslabs/aws-sdk-swift/blob/d1d18eefb7457ed27d416b372573a1f815004eb1/Sources/Services/AWSCloudTrail/models/Models.swift#L62)
+        if response.statusCode.rawValue > 299 {
+            let bodyData = try await response.body.readData()
+            response.body = .data(bodyData)
+        }
+
+        return try await clientProtocol.deserializeResponse(
+            operation: operation,
+            context: attributes,
+            response: response
+        )
+    }
+}

Sources/ClientRuntime/Networking/ServiceError.swift

@@ -12,7 +12,7 @@ public protocol ServiceError {
     var typeName: String? { get }
 
     /// The message for this error, if one was received.
-    var message: String? { get }
+    var message: String? { get set }
 }
 
 extension ServiceError {

Sources/ClientRuntime/Orchestrator/OrchestratorBuilder.swift

@@ -13,9 +13,12 @@ import protocol Smithy.RequestMessage
 import protocol Smithy.RequestMessageSerializer
 import protocol Smithy.ResponseMessage
 import protocol Smithy.ResponseMessageDeserializer
+import class SmithyHTTPAPI.HTTPResponse
 import struct SmithyHTTPAuthAPI.SelectedAuthScheme
 import struct SmithyRetriesAPI.RetryErrorInfo
 import protocol SmithyRetriesAPI.RetryStrategy
+import protocol SmithySerialization.ClientProtocol
+import struct SmithySerialization.Operation
 
 /// Builds an Orchestrator, combining runtime components, interceptors, serializers, and deserializers.
 ///
@@ -52,6 +55,18 @@ public class OrchestratorBuilder<
         return self
     }
 
+    @discardableResult
+    public func apply<ClientProtocol: SmithySerialization.ClientProtocol>(
+        _ operation: Operation<InputType, OutputType>,
+        _ clientProtocol: ClientProtocol
+    ) -> Self where ClientProtocol.RequestType == RequestType,
+                    ClientProtocol.ResponseType == ResponseType,
+                    ResponseType == HTTPResponse {
+        self.serialize(SchemaBodyMiddleware(operation, clientProtocol))
+        self.deserialize(SchemaDeserializeMiddleware(operation, clientProtocol))
+        return self
+    }
+
     /// - Parameter serializer: Function that performs part of request serialization
     /// - Returns: Builder
     @discardableResult