Codegenerate StructureShape, BlobShape, application, server and Python runtime #1403
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy | ||
|
|
||
| import software.amazon.smithy.rust.codegen.rustlang.CargoDependency | ||
| import software.amazon.smithy.rust.codegen.rustlang.CratesIo | ||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig | ||
|
|
||
| /** | ||
| * Object used *exclusively* in the runtime of the Python server, for separation concerns. | ||
| * Analogous to the companion object in [CargoDependency] and [ServerCargoDependency]; see its documentation for details. | ||
| * For a dependency that is used in the client, or in both the client and the server, use [CargoDependency] directly. | ||
| */ | ||
| object PythonServerCargoDependency { | ||
| val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.16"), features = setOf("extension-module")) | ||
| val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.16"), features = setOf("attributes", "tokio-runtime")) | ||
| val Tokio: CargoDependency = CargoDependency("tokio", CratesIo("1.0"), features = setOf("full")) | ||
| val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1")) | ||
| val Tower: CargoDependency = CargoDependency("tower", CratesIo("0.4")) | ||
| val TowerHttp: CargoDependency = CargoDependency("tower-http", CratesIo("0.3"), features = setOf("trace")) | ||
| val Hyper: CargoDependency = CargoDependency("hyper", CratesIo("0.14"), features = setOf("server", "http1", "http2", "tcp", "stream")) | ||
| val NumCpus: CargoDependency = CargoDependency("num_cpus", CratesIo("1.13")) | ||
|
|
||
| fun SmithyHttpServer(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server") | ||
| fun SmithyHttpServerPython(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server-python") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
|
|
||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy | ||
|
|
||
| import software.amazon.smithy.build.PluginContext | ||
| import software.amazon.smithy.model.shapes.ServiceShape | ||
| import software.amazon.smithy.model.shapes.StringShape | ||
| import software.amazon.smithy.model.shapes.StructureShape | ||
| import software.amazon.smithy.model.traits.EnumTrait | ||
| import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerEnumGenerator | ||
| import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerServiceGenerator | ||
| import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerStructureGenerator | ||
| import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenVisitor | ||
| import software.amazon.smithy.rust.codegen.server.smithy.protocols.ServerProtocolLoader | ||
| import software.amazon.smithy.rust.codegen.smithy.CodegenContext | ||
| import software.amazon.smithy.rust.codegen.smithy.DefaultPublicModules | ||
| import software.amazon.smithy.rust.codegen.smithy.RustCrate | ||
| import software.amazon.smithy.rust.codegen.smithy.SymbolVisitorConfig | ||
| import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.BuilderGenerator | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.CodegenTarget | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.implBlock | ||
| import software.amazon.smithy.rust.codegen.util.getTrait | ||
|
|
||
| /** | ||
| * Entrypoint for Python server-side code generation. This class will walk the in-memory model and | ||
| * generate all the needed types by calling the accept() function on the available shapes. | ||
| * | ||
| * This class inherits from [ServerCodegenVisitor] since it uses most of the functionlities of the super class | ||
| * and have to override the symbol provider with [PythonServerSymbolProvider]. | ||
| */ | ||
| class PythonServerCodegenVisitor(context: PluginContext, codegenDecorator: RustCodegenDecorator) : | ||
| ServerCodegenVisitor(context, codegenDecorator) { | ||
|
|
||
| init { | ||
|
There was a problem hiding this comment. Isn't this There was a problem hiding this comment. It is very similar, but it is needed to setup
and it is required to allow the inheritance to work and override the symbolProvider and everything else using it. There was a problem hiding this comment. We could have a function that given a symbol provider, sets up the rest of the stuff. |
||
| val symbolVisitorConfig = | ||
| SymbolVisitorConfig( | ||
| runtimeConfig = settings.runtimeConfig, | ||
| codegenConfig = settings.codegenConfig, | ||
| handleRequired = true | ||
| ) | ||
| val baseModel = baselineTransform(context.model) | ||
| val service = settings.getService(baseModel) | ||
| val (protocol, generator) = | ||
| ServerProtocolLoader( | ||
| codegenDecorator.protocols( | ||
| service.id, | ||
| ServerProtocolLoader.DefaultProtocols | ||
| ) | ||
| ) | ||
| .protocolFor(context.model, service) | ||
| protocolGeneratorFactory = generator | ||
| model = generator.transformModel(codegenDecorator.transformModel(service, baseModel)) | ||
| val baseProvider = PythonCodegenServerPlugin.baseSymbolProvider(model, service, symbolVisitorConfig) | ||
| // Override symbolProvider. | ||
| symbolProvider = | ||
| codegenDecorator.symbolProvider(generator.symbolProvider(model, baseProvider)) | ||
|
|
||
| // Override `codegenContext` which carries the symbolProvider. | ||
| codegenContext = CodegenContext(model, symbolProvider, service, protocol, settings, target = CodegenTarget.SERVER) | ||
|
|
||
| // Override `rustCrate` which carries the symbolProvider. | ||
| rustCrate = RustCrate(context.fileManifest, symbolProvider, DefaultPublicModules, settings.codegenConfig) | ||
| // Override `protocolGenerator` which carries the symbolProvider. | ||
| protocolGenerator = protocolGeneratorFactory.buildProtocolGenerator(codegenContext) | ||
| } | ||
|
|
||
| /** | ||
| * Structure Shape Visitor | ||
| * | ||
| * For each structure shape, generate: | ||
| * - A Rust structure for the shape ([StructureGenerator]). | ||
| * - `pyo3::PyClass` trait implementation. | ||
| * - A builder for the shape. | ||
| * | ||
| * This function _does not_ generate any serializers. | ||
| */ | ||
| override fun structureShape(shape: StructureShape) { | ||
| logger.info("[python-server-codegen] Generating a structure $shape") | ||
| rustCrate.useShapeWriter(shape) { writer -> | ||
| // Use Python specific structure generator that adds the #[pyclass] attribute | ||
| // and #[pymethods] implementation. | ||
| PythonServerStructureGenerator(model, codegenContext, symbolProvider, writer, shape).render(CodegenTarget.SERVER) | ||
| val builderGenerator = | ||
| BuilderGenerator(codegenContext.model, codegenContext.symbolProvider, shape) | ||
| builderGenerator.render(writer) | ||
| writer.implBlock(shape, symbolProvider) { | ||
| builderGenerator.renderConvenienceMethod(this) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * String Shape Visitor | ||
| * | ||
| * Although raw strings require no code generation, enums are actually [EnumTrait] applied to string shapes. | ||
| */ | ||
| override fun stringShape(shape: StringShape) { | ||
| logger.info("[rust-server-codegen] Generating an enum $shape") | ||
| shape.getTrait<EnumTrait>()?.also { enum -> | ||
| rustCrate.useShapeWriter(shape) { writer -> | ||
| PythonServerEnumGenerator(model, symbolProvider, writer, shape, enum, codegenContext.runtimeConfig).render() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Generate service-specific code for the model: | ||
| * - Serializers | ||
| * - Deserializers | ||
| * - Trait implementations | ||
| * - Protocol tests | ||
| * - Operation structures | ||
| * - Python operation handlers | ||
| */ | ||
| override fun serviceShape(shape: ServiceShape) { | ||
| logger.info("[python-server-codegen] Generating a service $shape") | ||
| PythonServerServiceGenerator( | ||
| rustCrate, | ||
| protocolGenerator, | ||
| protocolGeneratorFactory.support(), | ||
| protocolGeneratorFactory.protocol(codegenContext).httpBindingResolver, | ||
| codegenContext, | ||
| ) | ||
| .render() | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy | ||
|
|
||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig | ||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeType | ||
|
|
||
| /** | ||
| * Object used *exclusively* in the runtime of the Python server, for separation concerns. | ||
| * Analogous to the companion object in [RuntimeType] and [ServerRuntimeType]; see its documentation for details. | ||
| * For a runtime type that is used in the client, or in both the client and the server, use [RuntimeType] directly. | ||
| */ | ||
| object PythonServerRuntimeType { | ||
|
|
||
| fun SharedSocket(runtimeConfig: RuntimeConfig) = | ||
| RuntimeType("SharedSocket", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python") | ||
|
|
||
| fun Blob(runtimeConfig: RuntimeConfig) = | ||
| RuntimeType("Blob", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python::types") | ||
|
|
||
| fun PyError(runtimeConfig: RuntimeConfig) = | ||
| RuntimeType("Error", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy | ||
|
|
||
| import software.amazon.smithy.codegen.core.Symbol | ||
| import software.amazon.smithy.model.shapes.Shape | ||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeType | ||
| import software.amazon.smithy.rust.codegen.smithy.RustSymbolProvider | ||
| import software.amazon.smithy.rust.codegen.smithy.WrappingSymbolProvider | ||
| import software.amazon.smithy.rust.codegen.smithy.rustType | ||
|
|
||
| /** | ||
| * Input / output / error structures can refer to complex types like the ones implemented inside | ||
| * `aws_smithy_types` (a good example is `aws_smithy_types::Blob`). | ||
| * `aws_smithy_http_server_python::types` wraps those types that do not implement directly the | ||
| * `pyo3::PyClass` trait and cannot be shared safely with Python, providing an idiomatic Python / Rust API. | ||
| * | ||
| * This symbol provider ensures types not implementing `pyo3::PyClass` are swapped with their wrappers from | ||
| * `aws_smithy_http_server_python::types`. | ||
| */ | ||
| class PythonServerSymbolProvider(private val base: RustSymbolProvider) : | ||
| WrappingSymbolProvider(base) { | ||
|
|
||
| private val runtimeConfig = config().runtimeConfig | ||
|
|
||
| /** | ||
| * Convert a shape to a Symbol. | ||
| * | ||
| * Swap the shape's symbol if its associated type does not implement `pyo3::PyClass`. | ||
| */ | ||
| override fun toSymbol(shape: Shape): Symbol { | ||
| return when (base.toSymbol(shape).rustType()) { | ||
| RuntimeType.Blob(runtimeConfig).toSymbol().rustType() -> { | ||
crisidev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| PythonServerRuntimeType.Blob(runtimeConfig).toSymbol() | ||
| } | ||
| else -> { | ||
| base.toSymbol(shape) | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy.customizations | ||
|
|
||
| import software.amazon.smithy.rust.codegen.rustlang.Writable | ||
| import software.amazon.smithy.rust.codegen.rustlang.docs | ||
| import software.amazon.smithy.rust.codegen.rustlang.rust | ||
| import software.amazon.smithy.rust.codegen.rustlang.rustBlock | ||
| import software.amazon.smithy.rust.codegen.rustlang.writable | ||
| import software.amazon.smithy.rust.codegen.server.python.smithy.PythonServerRuntimeType | ||
| import software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOperationsDecorator | ||
| import software.amazon.smithy.rust.codegen.smithy.CodegenContext | ||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig | ||
| import software.amazon.smithy.rust.codegen.smithy.customize.CombinedCodegenDecorator | ||
| import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.LibRsCustomization | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.LibRsSection | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.ManifestCustomizations | ||
|
|
||
| /** | ||
| * Configure the [lib] section of `Cargo.toml`. | ||
| * | ||
| * [lib] | ||
| * name = "$CRATE_NAME" | ||
| * crate-type = ["cdylib"] | ||
| */ | ||
| class CdylibManifestDecorator : RustCodegenDecorator { | ||
| override val name: String = "CdylibDecorator" | ||
| override val order: Byte = 0 | ||
|
|
||
| override fun crateManifestCustomizations( | ||
| codegenContext: CodegenContext | ||
| ): ManifestCustomizations = | ||
| mapOf("lib" to mapOf("name" to codegenContext.settings.moduleName, "crate-type" to listOf("cdylib"))) | ||
| } | ||
|
|
||
| /** | ||
| * Add `pub use aws_smithy_http_server_python::types::$TYPE` to lib.rs. | ||
| */ | ||
| class PubUsePythonTypes(private val runtimeConfig: RuntimeConfig) : LibRsCustomization() { | ||
| override fun section(section: LibRsSection): Writable { | ||
| return when (section) { | ||
| is LibRsSection.Body -> writable { | ||
| docs("Re-exported Python types from supporting crates.") | ||
| rustBlock("pub mod python_types") { | ||
| rust("pub use #T;", PythonServerRuntimeType.Blob(runtimeConfig).toSymbol()) | ||
| } | ||
| } | ||
| else -> emptySection | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Decorator applying the customization from [PubUsePythonTypes] class. | ||
| */ | ||
| class PubUsePythonTypesDecorator : RustCodegenDecorator { | ||
| override val name: String = "PubUsePythonTypesDecorator" | ||
| override val order: Byte = 0 | ||
|
|
||
| override fun libRsCustomizations( | ||
| codegenContext: CodegenContext, | ||
| baseCustomizations: List<LibRsCustomization> | ||
| ): List<LibRsCustomization> { | ||
| return baseCustomizations + PubUsePythonTypes(codegenContext.runtimeConfig) | ||
| } | ||
| } | ||
|
|
||
| val DECORATORS = listOf( | ||
| /** | ||
| * Add the [InternalServerError] error to all operations. | ||
| * This is done because the Python interpreter can raise exceptions during execution | ||
| */ | ||
| AddInternalServerErrorToAllOperationsDecorator(), | ||
| // Add the [lib] section to Cargo.toml to configure the generation of the shared library: | ||
| CdylibManifestDecorator(), | ||
| // Add `pub use` of `aws_smithy_http_server_python::types`. | ||
| PubUsePythonTypesDecorator() | ||
| ) | ||
|
|
||
| // Combined codegen decorator for Python services. | ||
| class PythonServerCodegenDecorator : CombinedCodegenDecorator(DECORATORS) { | ||
| override val name: String = "PythonServerCodegenDecorator" | ||
| override val order: Byte = -1 | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Unrelated to this PR, but: why did we add this Python subproject inside the
codegen-server-testsubproject? As opposed to have it sitting besides it.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I preferred to use a subfolder as in my mind Python is just a subproduct of the Rust server codegen.