Codegenerate StructureShape, BlobShape, application, server and Python runtime

codegen-server-test/python/build.gradle.kts

@@ -16,7 +16,7 @@ val defaultRustFlags: String by project
 val defaultRustDocFlags: String by project
 val properties = PropertyRetriever(rootProject, project)
 
-val pluginName = "rust-server-codegen"
+val pluginName = "rust-server-codegen-python"
 val workingDirUnderBuildDir = "smithyprojections/codegen-server-test-python/"
 
 configure<software.amazon.smithy.gradle.SmithyExtension> {
@@ -39,13 +39,6 @@ dependencies {
 
 val allCodegenTests = listOf(
     CodegenTest("com.amazonaws.simple#SimpleService", "simple"),
-    CodegenTest("aws.protocoltests.restjson#RestJson", "rest_json"),
-    CodegenTest("aws.protocoltests.restjson.validation#RestJsonValidation", "rest_json_validation"),
-    CodegenTest("aws.protocoltests.json10#JsonRpc10", "json_rpc10"),
-    CodegenTest("aws.protocoltests.json#JsonProtocol", "json_rpc11"),
-    CodegenTest("aws.protocoltests.misc#MiscService", "misc"),
-    CodegenTest("com.amazonaws.ebs#Ebs", "ebs"),
-    CodegenTest("com.amazonaws.s3#AmazonS3", "s3"),
     CodegenTest("com.aws.example#PokemonService", "pokemon_service_sdk")
 )
 

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/RustCodegenServerPlugin.kt → codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonCodegenServerPlugin.kt

@@ -11,7 +11,6 @@ import software.amazon.smithy.codegen.core.ReservedWordSymbolProvider
 import software.amazon.smithy.model.Model
 import software.amazon.smithy.model.shapes.ServiceShape
 import software.amazon.smithy.rust.codegen.rustlang.RustReservedWordSymbolProvider
-import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenVisitor
 import software.amazon.smithy.rust.codegen.smithy.BaseSymbolMetadataProvider
 import software.amazon.smithy.rust.codegen.smithy.DefaultConfig
 import software.amazon.smithy.rust.codegen.smithy.EventStreamSymbolProvider
@@ -28,7 +27,7 @@ import java.util.logging.Logger
  *  `resources/META-INF.services/software.amazon.smithy.build.SmithyBuildPlugin` refers to this class by name which
  *  enables the smithy-build plugin to invoke `execute` with all of the Smithy plugin context + models.
  */
-class RustCodegenServerPlugin : SmithyBuildPlugin {
+class PythonCodegenServerPlugin : SmithyBuildPlugin {
     private val logger = Logger.getLogger(javaClass.name)
 
     override fun getName(): String = "rust-server-codegen-python"
@@ -44,8 +43,8 @@ class RustCodegenServerPlugin : SmithyBuildPlugin {
         val codegenDecorator = CombinedCodegenDecorator.fromClasspath(context)
 
         // ServerCodegenVisitor is the main driver of code generation that traverses the model and generates code
-        logger.info("Loaded plugin to generate Rust/Python bindings for the server SSDK")
-        ServerCodegenVisitor(context, codegenDecorator).execute()
+        logger.warning("Loaded plugin to generate Rust/Python bindings for the server SSDK for projection ${context.projectionName}")
+        PythonServerCodegenVisitor(context, codegenDecorator).execute()
     }
 
     companion object {
@@ -74,5 +73,8 @@ class RustCodegenServerPlugin : SmithyBuildPlugin {
                 // Rename shapes that clash with Rust reserved words & and other SDK specific features e.g. `send()` cannot
                 // be the name of an operation input
                 .let { RustReservedWordSymbolProvider(it, model) }
+                // Rename a set of symbols that do not implement `PyClass` and have been wrapped inside
+                // `aws_smithy_http_server_python::types`.
+                .let { PythonServerSymbolProvider(it, model) }
     }
 }

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.rust.codegen.server.python.smithy
+
+import software.amazon.smithy.rust.codegen.rustlang.CargoDependency
+import software.amazon.smithy.rust.codegen.rustlang.CratesIo
+import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig
+
+/**
+ * Object used *exclusively* in the runtime of the Python server, for separation concerns.
+ * Analogous to the companion object in [CargoDependency] and [ServerCargoDependency]; see its documentation for details.
+ * For a dependency that is used in the client, or in both the client and the server, use [CargoDependency] directly.
+ */
+object PythonServerCargoDependency {
+    val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.16"), features = setOf("extension-module"))
+    val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.16"), features = setOf("attributes", "tokio-runtime"))
+    val Tokio: CargoDependency = CargoDependency("tokio", CratesIo("1.0"), features = setOf("full"))
+    val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1"))
+    val Tower: CargoDependency = CargoDependency("tower", CratesIo("0.4"))
+    val TowerHttp: CargoDependency = CargoDependency("tower-http", CratesIo("0.3"), features = setOf("trace"))
+    val Hyper: CargoDependency = CargoDependency("hyper", CratesIo("0.14"), features = setOf("server", "http1", "http2", "tcp", "stream"))
+    val NumCpus: CargoDependency = CargoDependency("num_cpus", CratesIo("1.13"))
+
+    fun SmithyHttpServer(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server")
+    fun SmithyHttpServerPython(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server-python")
+}

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCodegenVisitor.kt

@@ -0,0 +1,132 @@
+
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.rust.codegen.server.python.smithy
+
+import software.amazon.smithy.build.PluginContext
+import software.amazon.smithy.model.shapes.ServiceShape
+import software.amazon.smithy.model.shapes.StringShape
+import software.amazon.smithy.model.shapes.StructureShape
+import software.amazon.smithy.model.traits.EnumTrait
+import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerEnumGenerator
+import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerServiceGenerator
+import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerStructureGenerator
+import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenVisitor
+import software.amazon.smithy.rust.codegen.server.smithy.protocols.ServerProtocolLoader
+import software.amazon.smithy.rust.codegen.smithy.CodegenContext
+import software.amazon.smithy.rust.codegen.smithy.CodegenMode
+import software.amazon.smithy.rust.codegen.smithy.DefaultPublicModules
+import software.amazon.smithy.rust.codegen.smithy.RustCrate
+import software.amazon.smithy.rust.codegen.smithy.SymbolVisitorConfig
+import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator
+import software.amazon.smithy.rust.codegen.smithy.generators.BuilderGenerator
+import software.amazon.smithy.rust.codegen.smithy.generators.CodegenTarget
+import software.amazon.smithy.rust.codegen.smithy.generators.implBlock
+import software.amazon.smithy.rust.codegen.util.getTrait
+
+/**
+ * Entrypoint for Python server-side code generation. This class will walk the in-memory model and
+ * generate all the needed types by calling the accept() function on the available shapes.
+ *
+ * This class inherits from [ServerCodegenVisitor] since it uses most of the functionlities of the super class
+ * and have to override the symbol provider with [PythonServerSymbolProvider].
+ */
+class PythonServerCodegenVisitor(context: PluginContext, private val codegenDecorator: RustCodegenDecorator) :
+    ServerCodegenVisitor(context, codegenDecorator) {
+
+    init {
+        val symbolVisitorConfig =
+            SymbolVisitorConfig(
+                runtimeConfig = settings.runtimeConfig,
+                codegenConfig = settings.codegenConfig,
+                handleRequired = true
+            )
+        val baseModel = baselineTransform(context.model)
+        val service = settings.getService(baseModel)
+        val (protocol, generator) =
+            ServerProtocolLoader(
+                codegenDecorator.protocols(
+                    service.id,
+                    ServerProtocolLoader.DefaultProtocols
+                )
+            )
+                .protocolFor(context.model, service)
+        protocolGeneratorFactory = generator
+        model = generator.transformModel(codegenDecorator.transformModel(service, baseModel))
+        val baseProvider = PythonCodegenServerPlugin.baseSymbolProvider(model, service, symbolVisitorConfig)
+        // Override symbolProvider.
+        symbolProvider =
+            codegenDecorator.symbolProvider(generator.symbolProvider(model, baseProvider))
+
+        // Override `codegenContext` which carries the symbolProvider.
+        codegenContext = CodegenContext(model, symbolProvider, service, protocol, settings, mode = CodegenMode.Server)
+
+        // Override `rustCrate` which carries the symbolProvider.
+        rustCrate = RustCrate(context.fileManifest, symbolProvider, DefaultPublicModules, settings.codegenConfig)
+        // Override `protocolGenerator` which carries the symbolProvider.
+        protocolGenerator = protocolGeneratorFactory.buildProtocolGenerator(codegenContext)
+    }
+
+    /**
+     * Structure Shape Visitor
+     *
+     * For each structure shape, generate:
+     * - A Rust structure for the shape ([StructureGenerator]).
+     * - `pyo3::PyClass` trait implementation.
+     * - A builder for the shape.
+     *
+     * This function _does not_ generate any serializers.
+     */
+    override fun structureShape(shape: StructureShape) {
+        logger.info("[python-server-codegen] Generating a structure $shape")
+        rustCrate.useShapeWriter(shape) { writer ->
+            // Use Python specific structure generator that adds the #[pyclass] attribute
+            // and #[pymethods] implementation.
+            PythonServerStructureGenerator(model, codegenContext, symbolProvider, writer, shape).render(CodegenTarget.SERVER)
+            val builderGenerator =
+                BuilderGenerator(codegenContext.model, codegenContext.symbolProvider, shape)
+            builderGenerator.render(writer)
+            writer.implBlock(shape, symbolProvider) {
+                builderGenerator.renderConvenienceMethod(this)
+            }
+        }
+    }
+
+    /**
+     * Enum Shape Visitor
+     *
+     * Although raw strings require no code generation, enums are actually [EnumTrait] applied to string shapes.
+     */
+    override fun stringShape(shape: StringShape) {
+        logger.info("[rust-server-codegen] Generating an enum $shape")
+        shape.getTrait<EnumTrait>()?.also { enum ->
+            rustCrate.useShapeWriter(shape) { writer ->
+                PythonServerEnumGenerator(model, symbolProvider, writer, shape, enum, codegenContext.runtimeConfig).render()
+            }
+        }
+    }
+
+    /**
+     * Generate service-specific code for the model:
+     * - Serializers
+     * - Deserializers
+     * - Trait implementations
+     * - Protocol tests
+     * - Operation structures
+     * - Python operation handlers
+     */
+    override fun serviceShape(shape: ServiceShape) {
+        logger.info("[python-server-codegen] Generating a service $shape")
+        PythonServerServiceGenerator(
+            rustCrate,
+            protocolGenerator,
+            protocolGeneratorFactory.support(),
+            protocolGeneratorFactory.protocol(codegenContext).httpBindingResolver,
+            codegenContext,
+        )
+            .render()
+    }
+}

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerRuntimeType.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.rust.codegen.server.python.smithy
+
+import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig
+import software.amazon.smithy.rust.codegen.smithy.RuntimeType
+
+/**
+ * Object used *exclusively* in the runtime of the Python server, for separation concerns.
+ * Analogous to the companion object in [RuntimeType] and [ServerRuntimeType]; see its documentation for details.
+ * For a runtime type that is used in the client, or in both the client and the server, use [RuntimeType] directly.
+ */
+object PythonServerRuntimeType {
+
+    fun SharedSocket(runtimeConfig: RuntimeConfig) =
+        RuntimeType("SharedSocket", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python")
+
+    fun Blob(runtimeConfig: RuntimeConfig) =
+        RuntimeType("Blob", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python::types")
+
+    fun PyError(runtimeConfig: RuntimeConfig) =
+        RuntimeType("Error", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python")
+}

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerSymbolProvider.kt

@@ -0,0 +1,55 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.rust.codegen.server.python.smithy
+
+import software.amazon.smithy.codegen.core.Symbol
+import software.amazon.smithy.model.Model
+import software.amazon.smithy.model.shapes.Shape
+import software.amazon.smithy.rust.codegen.rustlang.RustMetadata
+import software.amazon.smithy.rust.codegen.rustlang.RustType
+import software.amazon.smithy.rust.codegen.smithy.RustSymbolProvider
+import software.amazon.smithy.rust.codegen.smithy.WrappingSymbolProvider
+import software.amazon.smithy.rust.codegen.smithy.meta
+import software.amazon.smithy.rust.codegen.smithy.rustType
+
+/**
+ * Input / output / error structures can refer to complex types like the ones implemented inside
+ * `aws_smithy_types` (a good example is `aws_smithy_types::Blob`).
+ * `aws_smithy_http_server_python::types` wraps those types that do not implement directly the
+ * `pyo3::PyClass` trait and cannot be share safely to Python, providing an idiomatic Python / Rust API.
+ *
+ * This symbol provider ensures types not implemeting `pyo3::PyClass` are swapped with their wrappers from
+ * `aws_smithy_http_server_python::types`.
+ */
+class PythonServerSymbolProvider(private val base: RustSymbolProvider, private val model: Model) :
+    WrappingSymbolProvider(base) {
+
+    /**
+     * Convert a shape to a Symbol.
+     *
+     * Swap the symbol if the shape's symbol does not implement `pyo3::PyClass`.
+     */
+    override fun toSymbol(shape: Shape): Symbol {
+        return when (base.toSymbol(shape).fullName) {
+            "aws_smithy_types::Blob" -> {
+                buildSymbol("Blob", "aws_smithy_http_server_python::types")
+            }
+            else -> {
+                base.toSymbol(shape)
+            }
+        }
+    }
+
+    /**
+     * Create a new symbol based on its name, namespace and metadata.
+     */
+    private fun buildSymbol(name: String, namespace: String, public: Boolean = false): Symbol =
+        Symbol.builder()
+            .name(name)
+            .namespace(namespace, "::")
+            .meta(RustMetadata(public = public))
+            .rustType(RustType.Opaque(name ?: "", namespace = namespace)).build()
+}

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.rust.codegen.server.python.smithy.customizations
+
+import software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOpsDecorator
+import software.amazon.smithy.rust.codegen.smithy.CodegenContext
+import software.amazon.smithy.rust.codegen.smithy.customize.CombinedCodegenDecorator
+import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator
+import software.amazon.smithy.rust.codegen.smithy.generators.ManifestCustomizations
+
+/**
+ * Configure the [lib] section to Cargo.toml.
+ *
+ * [lib]
+ * name = "$CRATE_NAME"
+ * crate-type = ["cdylib"]
+ */
+class CdylibManifestDecorator : RustCodegenDecorator {
+    override val name: String = "CdylibDecorator"
+    override val order: Byte = 0
+
+    override fun crateManifestCustomizations(
+        codegenContext: CodegenContext
+    ): ManifestCustomizations =
+        mapOf("lib" to mapOf("name" to codegenContext.settings.moduleName, "crate-type" to listOf("cdylib")))
+}
+
+val DECORATORS = listOf(
+    /**
+     * Add the [InternalServerError] error to all operations.
+     * This is done because the Python interpreter can raise exceptions during execution
+     * and we cannot guarantee infallible execution of operations.
+     */
+    AddInternalServerErrorToAllOpsDecorator(),
+    // Add the [lib] section to Cargo.toml to configure the generation of the shared library:
+    CdylibManifestDecorator()
+)
+
+// Combined codegen decorator for Python services.
+class PythonServerCodegenDecorator : CombinedCodegenDecorator(DECORATORS) {
+    override val name: String = "PythonServerCodegenDecorator"
+    override val order: Byte = -1
+}