From 1a2dcd3afa917f29d962236fb1ebae5bb62d4e83 Mon Sep 17 00:00:00 2001 From: Matteo Bigoi <1781140+crisidev@users.noreply.github.com> Date: Mon, 17 Apr 2023 17:19:12 +0100 Subject: [PATCH] [Python] Automatically generate stubs (#2576) ## Motivation and Context We want to automatically generate stubs in the codegen diff to ensure they can be reviewed and have a simple way to generate and include the stubs inside the Maturin wheel. ## Description The Python example has been moved to the `examples` folder and refactored. The refactoring ensures the script `stubgen.py` is included in the codegeneration of the SDK crate. The script is later used to generate stubs automatically during testing and can be used by customers to add their own stubs before the Maturin build ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Signed-off-by: Bigo <1781140+crisidev@users.noreply.github.com> Co-authored-by: Burak --- .../rust/codegen/core/rustlang/RustWriter.kt | 1 + .../rust/codegen/core/smithy/RuntimeType.kt | 2 + codegen-server-test/python/build.gradle.kts | 17 +++- .../smithy/PythonServerCargoDependency.kt | 4 +- .../PythonServerCodegenDecorator.kt | 28 ++++++ .../generators/PythonServerModuleGenerator.kt | 40 ++++++-- .../RustCrateInlineModuleComposingWriter.kt | 4 +- .../examples => examples/python}/.gitignore | 0 .../examples => examples/python}/Cargo.toml | 0 .../examples => examples/python}/Makefile | 14 ++- examples/python/README.md | 94 +++++++++++++++++++ .../examples => examples/python}/mypy.ini | 0 .../python}/pokemon-service-test/Cargo.toml | 14 +-- .../pokemon-service-test/tests/helpers.rs | 0 .../tests/simple_integration_test.rs | 0 .../tests/testdata/localhost.crt | 0 .../tests/testdata/localhost.key | 0 .../python}/pokemon_service.py | 0 .../aws-smithy-http-server-python/Cargo.toml | 14 +-- .../examples/README.md | 52 ---------- .../aws-smithy-http-server-python/src/tls.rs | 6 +- .../src/tls/listener.rs | 2 +- .../{examples => }/stubgen.py | 75 +++++++-------- .../aws-smithy-http-server-python/stubgen.sh | 48 ++++++++++ .../{examples => }/stubgen_test.py | 0 tools/ci-scripts/check-server-python-e2e-test | 2 +- tools/ci-scripts/codegen-diff/diff_lib.py | 13 +-- 27 files changed, 291 insertions(+), 139 deletions(-) rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/.gitignore (100%) rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/Cargo.toml (100%) rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/Makefile (90%) create mode 100644 examples/python/README.md rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/mypy.ini (100%) rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/pokemon-service-test/Cargo.toml (53%) rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/pokemon-service-test/tests/helpers.rs (100%) rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/pokemon-service-test/tests/simple_integration_test.rs (100%) rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/pokemon-service-test/tests/testdata/localhost.crt (100%) rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/pokemon-service-test/tests/testdata/localhost.key (100%) rename {rust-runtime/aws-smithy-http-server-python/examples => examples/python}/pokemon_service.py (100%) delete mode 100644 rust-runtime/aws-smithy-http-server-python/examples/README.md rename rust-runtime/aws-smithy-http-server-python/{examples => }/stubgen.py (88%) create mode 100755 rust-runtime/aws-smithy-http-server-python/stubgen.sh rename rust-runtime/aws-smithy-http-server-python/{examples => }/stubgen_test.py (100%) diff --git a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt index cf2ff4b5d5..8de5cdd59c 100644 --- a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt +++ b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt @@ -469,6 +469,7 @@ class RustWriter private constructor( devDependenciesOnly = true, ) fileName == "package.json" -> rawWriter(fileName, debugMode = debugMode) + fileName == "stubgen.sh" -> rawWriter(fileName, debugMode = debugMode) else -> RustWriter(fileName, namespace, debugMode = debugMode) } } diff --git a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt index 7e53055195..6be653bd59 100644 --- a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt +++ b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt @@ -96,6 +96,8 @@ data class RuntimeConfig( val crateSrcPrefix: String = cratePrefix.replace("-", "_") + fun runtimeCratesPath(): String? = runtimeCrateLocation.path + fun smithyRuntimeCrate( runtimeCrateName: String, optional: Boolean = false, diff --git a/codegen-server-test/python/build.gradle.kts b/codegen-server-test/python/build.gradle.kts index 87ff7cc5bd..e70e2b0e5d 100644 --- a/codegen-server-test/python/build.gradle.kts +++ b/codegen-server-test/python/build.gradle.kts @@ -66,7 +66,7 @@ val allCodegenTests = "../../codegen-core/common-test-models".let { commonModels "rest_json_extras", imports = listOf("$commonModels/rest-json-extras.smithy"), ), - // TODO(https://github.com/awslabs/smithy-rs/issues/2551) + // TODO(https://github.com/awslabs/smithy-rs/issues/2477) // CodegenTest( // "aws.protocoltests.restjson.validation#RestJsonValidation", // "rest_json_validation", @@ -104,6 +104,21 @@ project.registerGenerateSmithyBuildTask(rootProject, pluginName, allCodegenTests project.registerGenerateCargoWorkspaceTask(rootProject, pluginName, allCodegenTests, workingDirUnderBuildDir) project.registerGenerateCargoConfigTomlTask(buildDir.resolve(workingDirUnderBuildDir)) +tasks.register("stubs") { + description = "Generate Python stubs for all models" + dependsOn("assemble") + + doLast { + allCodegenTests.forEach { test -> + val crateDir = "$buildDir/$workingDirUnderBuildDir/${test.module}/$pluginName" + val moduleName = test.module.replace("-", "_") + exec { + commandLine("bash", "$crateDir/stubgen.sh", moduleName, "$crateDir/Cargo.toml", "$crateDir/python/$moduleName") + } + } + } +} + tasks["smithyBuildJar"].dependsOn("generateSmithyBuild") tasks["assemble"].finalizedBy("generateCargoWorkspace") diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt index 911edb893e..2782e44b94 100644 --- a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt @@ -15,8 +15,8 @@ import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig * For a dependency that is used in the client, or in both the client and the server, use [CargoDependency] directly. */ object PythonServerCargoDependency { - val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.17")) - val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.17"), features = setOf("attributes", "tokio-runtime")) + val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.18")) + val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.18"), features = setOf("attributes", "tokio-runtime")) val Tokio: CargoDependency = CargoDependency("tokio", CratesIo("1.20.1"), features = setOf("full")) val TokioStream: CargoDependency = CargoDependency("tokio-stream", CratesIo("0.1.12")) val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1")) diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt index 9e353db2cd..ff4b6be482 100644 --- a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt @@ -23,6 +23,7 @@ import software.amazon.smithy.rust.codegen.server.python.smithy.generators.Pytho import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenContext import software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOperationsDecorator import software.amazon.smithy.rust.codegen.server.smithy.customize.ServerCodegenDecorator +import java.io.File /** * Configure the [lib] section of `Cargo.toml`. @@ -194,6 +195,31 @@ class PyTypedMarkerDecorator : ServerCodegenDecorator { } } +/** + * Copies the stubgen scripts to the generated crate root. + * + * The shell script `stubgen.sh` runs a quick build and uses `stubgen.py` to generate mypy compatibile + * types stubs for the project. + */ +class AddStubgenScriptDecorator : ServerCodegenDecorator { + override val name: String = "AddStubgenScriptDecorator" + override val order: Byte = 0 + + override fun extras(codegenContext: ServerCodegenContext, rustCrate: RustCrate) { + val runtimeCratesPath = codegenContext.runtimeConfig.runtimeCratesPath() + val stubgenPythonLocation = "$runtimeCratesPath/aws-smithy-http-server-python/stubgen.py" + val stubgenPythonContent = File(stubgenPythonLocation).readText(Charsets.UTF_8) + rustCrate.withFile("stubgen.py") { + writeWithNoFormatting("$stubgenPythonContent") + } + val stubgenShellLocation = "$runtimeCratesPath/aws-smithy-http-server-python/stubgen.sh" + val stubgenShellContent = File(stubgenShellLocation).readText(Charsets.UTF_8) + rustCrate.withFile("stubgen.sh") { + writeWithNoFormatting("$stubgenShellContent") + } + } +} + val DECORATORS = arrayOf( /** * Add the [InternalServerError] error to all operations. @@ -214,4 +240,6 @@ val DECORATORS = arrayOf( InitPyDecorator(), // Generate `py.typed` for the Python source. PyTypedMarkerDecorator(), + // Generate scripts for stub generation. + AddStubgenScriptDecorator(), ) diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt index 5baf6e83ea..577999a724 100644 --- a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt @@ -11,7 +11,9 @@ import software.amazon.smithy.model.shapes.ResourceShape import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.shapes.Shape import software.amazon.smithy.model.shapes.UnionShape +import software.amazon.smithy.rust.codegen.core.Version import software.amazon.smithy.rust.codegen.core.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.core.rustlang.rust import software.amazon.smithy.rust.codegen.core.rustlang.rustBlockTemplate import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate import software.amazon.smithy.rust.codegen.core.smithy.RustCrate @@ -51,6 +53,8 @@ class PythonServerModuleGenerator( renderPyTlsTypes() renderPyLambdaTypes() renderPyApplicationType() + renderCodegenVersion() + rust("Ok(())") } } } @@ -62,13 +66,23 @@ class PythonServerModuleGenerator( let input = #{pyo3}::types::PyModule::new(py, "input")?; let output = #{pyo3}::types::PyModule::new(py, "output")?; let error = #{pyo3}::types::PyModule::new(py, "error")?; - let model = #{pyo3}::types::PyModule::new(py, "model")?; """, *codegenScope, ) + // The `model` type section can be unused in models like `simple`, so we accommodate for it. + var visitedModelType = false serviceShapes.forEach { shape -> val moduleType = moduleType(shape) if (moduleType != null) { + if (moduleType == "model" && !visitedModelType) { + rustTemplate( + """ + let model = #{pyo3}::types::PyModule::new(py, "model")?; + """, + *codegenScope, + ) + visitedModelType = true + } when (shape) { is UnionShape -> rustTemplate( """ @@ -93,11 +107,18 @@ class PythonServerModuleGenerator( m.add_submodule(output)?; #{pyo3}::py_run!(py, error, "import sys; sys.modules['$libName.error'] = error"); m.add_submodule(error)?; - #{pyo3}::py_run!(py, model, "import sys; sys.modules['$libName.model'] = model"); - m.add_submodule(model)?; """, *codegenScope, ) + if (visitedModelType) { + rustTemplate( + """ + #{pyo3}::py_run!(py, model, "import sys; sys.modules['$libName.model'] = model"); + m.add_submodule(model)?; + """, + *codegenScope, + ) + } } // Render wrapper types that are substituted to the ones coming from `aws_smithy_types`. @@ -211,13 +232,12 @@ class PythonServerModuleGenerator( // Render Python application type. private fun RustWriter.renderPyApplicationType() { - rustTemplate( - """ - m.add_class::()?; - Ok(()) - """, - *codegenScope, - ) + rust("""m.add_class::()?;""") + } + + // Render the codegeneration version as module attribute. + private fun RustWriter.renderCodegenVersion() { + rust("""m.add("CODEGEN_VERSION", "${Version.crateVersion()}")?;""") } // Convert to symbol and check the namespace to figure out where they should be imported from. diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCrateInlineModuleComposingWriter.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCrateInlineModuleComposingWriter.kt index 32abe41d69..96b3b5739c 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCrateInlineModuleComposingWriter.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCrateInlineModuleComposingWriter.kt @@ -335,7 +335,9 @@ class InnerModule(private val moduleDocProvider: ModuleDocProvider, debugMode: B inlineWriter } else { check(inlineModuleAndWriter.inlineModule == lookForModule) { - "The two inline modules have the same name but different attributes on them." + """The two inline modules have the same name but different attributes on them: + 1) ${inlineModuleAndWriter.inlineModule} + 2) $lookForModule""" } inlineModuleAndWriter.writer diff --git a/rust-runtime/aws-smithy-http-server-python/examples/.gitignore b/examples/python/.gitignore similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/.gitignore rename to examples/python/.gitignore diff --git a/rust-runtime/aws-smithy-http-server-python/examples/Cargo.toml b/examples/python/Cargo.toml similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/Cargo.toml rename to examples/python/Cargo.toml diff --git a/rust-runtime/aws-smithy-http-server-python/examples/Makefile b/examples/python/Makefile similarity index 90% rename from rust-runtime/aws-smithy-http-server-python/examples/Makefile rename to examples/python/Makefile index 3b73f08ca6..56213fe289 100644 --- a/rust-runtime/aws-smithy-http-server-python/examples/Makefile +++ b/examples/python/Makefile @@ -35,25 +35,23 @@ install-wheel: find $(WHEELS) -type f -name '*.whl' | xargs python3 -m pip install --user --force-reinstall generate-stubs: - python3 $(CUR_DIR)/stubgen.py pokemon_service_server_sdk $(SERVER_SDK_DST)/python/pokemon_service_server_sdk + bash $(SERVER_SDK_DST)/stubgen.sh pokemon_service_server_sdk $(SERVER_SDK_DST)/Cargo.toml $(SERVER_SDK_DST)/python/pokemon_service_server_sdk build: codegen + $(MAKE) generate-stubs $(MAKE) build-wheel - $(MAKE) install-wheel + +release: codegen $(MAKE) generate-stubs $(MAKE) build-wheel-release - $(MAKE) install-wheel run: build python3 $(CUR_DIR)/pokemon_service.py -py-check: build +py-check: install-wheel python3 -m mypy pokemon_service.py -py-test: - python3 stubgen_test.py - -test: build py-check py-test +test: build py-check cargo test clippy: codegen diff --git a/examples/python/README.md b/examples/python/README.md new file mode 100644 index 0000000000..ebe6cfb39e --- /dev/null +++ b/examples/python/README.md @@ -0,0 +1,94 @@ +# Smithy Rust/Python Server SDK example + +This folder contains an example service called Pokémon Service used to showcase +the service framework Python bindings capabilities and to run benchmarks. + +The Python implementation of the service can be found inside +[pokemon_service.py](./pokemon_service.py). + +* [Build](#build) + * [Build dependencies](#build-dependencies) + * [Makefile](#makefile) + * [Python stub generation](#python-stub-generation) +* [Run](#run) +* [Test](#test) +* [Uvloop](#uvloop) +* [MacOs](#macos) + +## Build + +Since this example requires both the server and client SDK to be code-generated +from their [model](/codegen-server-test/model/pokemon.smithy), a Makefile is +provided to build and run the service. Just run `make build` to prepare the first +build. + +### Build dependencies + +Ensure these dependencies are installed. + +``` +pip install maturin uvloop aiohttp mypy +``` + +### Makefile + +The build logic is drive by the Makefile: + +* `make codegen`: run the codegenerator. +* `make build`: build the Maturin package in debug mode (includes type stubs + generation). +* `make release`: build the Maturin package in release mode (includes type stub + generation). +* `make install`: install the latest release or debug build. +* `make run`: run the example server. +* `make test`: run the end-to-end integration test. + +### Python stub generation + +We support the generation of mypy python stubs and every SDK crate ships with +a script called `stubgen.sh`. **Note that the script is not called +automatically as part of the build**. We suggest users to call it after code generation. +It will do first compilation of the crate, generate the types and exit. + +The script takes some command line arguments: + +``` +./stubgen.sh module_name manifest_path output_directory +``` + +* module_name: name of the Python module to generate stubs for, IE `pokemon_service_server_sdk`. +* manifest_path: path for the crate manifest used to build the types. +* output_directory: directory where to generate the stub hierarchy. **This + directory should be a folder `python/$module_name` in the root of the Maturin package.** + +## Run + +`make run` can be used to start the Pokémon service on `http://localhost:13734`. + +## Test + +`make test` can be used to spawn the Python service and run some simple integration +tests against it. + +More info can be found in the `tests` folder of `pokemon-service-test` package. + +## Uvloop + +The server can depend on [uvloop](https://pypi.org/project/uvloop/) for a +faster event loop implementation. Uvloop can be installed with your favourite +package manager or by using pip: + +```sh +pip instal uvloop +``` + +and it will be automatically used instead of the standard library event loop if +it is found in the dependencies' closure. + +## MacOs + +To compile and test on MacOs, please follow the official PyO3 guidelines on how +to [configure your linker](https://pyo3.rs/latest/building_and_distribution.html?highlight=rustflags#macos). + +Please note that the `.cargo/config.toml` with linkers override can be local to +your project. diff --git a/rust-runtime/aws-smithy-http-server-python/examples/mypy.ini b/examples/python/mypy.ini similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/mypy.ini rename to examples/python/mypy.ini diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/Cargo.toml b/examples/python/pokemon-service-test/Cargo.toml similarity index 53% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/Cargo.toml rename to examples/python/pokemon-service-test/Cargo.toml index 6edcebf160..8dbe8c6974 100644 --- a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/Cargo.toml +++ b/examples/python/pokemon-service-test/Cargo.toml @@ -9,15 +9,15 @@ description = "Run tests against the Python server implementation" [dev-dependencies] rand = "0.8" async-stream = "0.3" -command-group = "1.0" +command-group = "2.1.0" tokio = { version = "1.20.1", features = ["full"] } -serial_test = "0.9.0" +serial_test = "2.0.0" rustls-pemfile = "1.0.1" -tokio-rustls = "0.23.4" -hyper-rustls = { version = "0.23.0", features = ["http2"] } +tokio-rustls = "0.24.0" +hyper-rustls = { version = "0.24.0", features = ["http2"] } # Local paths -aws-smithy-client = { path = "../../../aws-smithy-client/", features = ["rustls"] } -aws-smithy-http = { path = "../../../aws-smithy-http/" } -aws-smithy-types = { path = "../../../aws-smithy-types/" } +aws-smithy-client = { path = "../../../rust-runtime/aws-smithy-client/", features = ["rustls"] } +aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http/" } +aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types/" } pokemon-service-client = { path = "../pokemon-service-client/" } diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/helpers.rs b/examples/python/pokemon-service-test/tests/helpers.rs similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/helpers.rs rename to examples/python/pokemon-service-test/tests/helpers.rs diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/simple_integration_test.rs b/examples/python/pokemon-service-test/tests/simple_integration_test.rs similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/simple_integration_test.rs rename to examples/python/pokemon-service-test/tests/simple_integration_test.rs diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/testdata/localhost.crt b/examples/python/pokemon-service-test/tests/testdata/localhost.crt similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/testdata/localhost.crt rename to examples/python/pokemon-service-test/tests/testdata/localhost.crt diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/testdata/localhost.key b/examples/python/pokemon-service-test/tests/testdata/localhost.key similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/testdata/localhost.key rename to examples/python/pokemon-service-test/tests/testdata/localhost.key diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon_service.py b/examples/python/pokemon_service.py similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon_service.py rename to examples/python/pokemon_service.py diff --git a/rust-runtime/aws-smithy-http-server-python/Cargo.toml b/rust-runtime/aws-smithy-http-server-python/Cargo.toml index 69e9577160..738b86418b 100644 --- a/rust-runtime/aws-smithy-http-server-python/Cargo.toml +++ b/rust-runtime/aws-smithy-http-server-python/Cargo.toml @@ -22,9 +22,9 @@ bytes = "1.2" futures = "0.3" http = "0.2" hyper = { version = "0.14.20", features = ["server", "http1", "http2", "tcp", "stream"] } -tls-listener = { version = "0.5.1", features = ["rustls", "hyper-h2"] } +tls-listener = { version = "0.7.0", features = ["rustls", "hyper-h2"] } rustls-pemfile = "1.0.1" -tokio-rustls = "0.23.4" +tokio-rustls = "0.24.0" lambda_http = { version = "0.7.1" } # There is a breaking change in `lambda_runtime` between `0.7.0` and `0.7.1`, # and `lambda_http` depends on `0.7` which by default resolves to `0.7.1` but in our CI @@ -34,10 +34,10 @@ lambda_runtime = { version = "0.7.1" } num_cpus = "1.13.1" parking_lot = "0.12.1" pin-project-lite = "0.2" -pyo3 = "0.17.0" -pyo3-asyncio = { version = "0.17.0", features = ["tokio-runtime"] } +pyo3 = "0.18.2" +pyo3-asyncio = { version = "0.18.0", features = ["tokio-runtime"] } signal-hook = { version = "0.3.14", features = ["extended-siginfo"] } -socket2 = { version = "0.4.4", features = ["all"] } +socket2 = { version = "0.5.2", features = ["all"] } thiserror = "1.0.32" tokio = { version = "1.20.1", features = ["full"] } tokio-stream = "0.1" @@ -51,9 +51,9 @@ pretty_assertions = "1" futures-util = { version = "0.3.16", default-features = false } tower-test = "0.4" tokio-test = "0.4" -pyo3-asyncio = { version = "0.17.0", features = ["testing", "attributes", "tokio-runtime", "unstable-streams"] } +pyo3-asyncio = { version = "0.18.0", features = ["testing", "attributes", "tokio-runtime", "unstable-streams"] } rcgen = "0.10.0" -hyper-rustls = { version = "0.23.1", features = ["http2"] } +hyper-rustls = { version = "0.24.0", features = ["http2"] } # PyO3 Asyncio tests cannot use Cargo's default testing harness because `asyncio` # wants to control the main thread. So we need to use testing harness provided by `pyo3_asyncio` diff --git a/rust-runtime/aws-smithy-http-server-python/examples/README.md b/rust-runtime/aws-smithy-http-server-python/examples/README.md deleted file mode 100644 index f765f4b57b..0000000000 --- a/rust-runtime/aws-smithy-http-server-python/examples/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Smithy Rust/Python Server SDK example - -This folder contains an example service called Pokémon Service used to showcase -the service framework Python bindings capabilities and to run benchmarks. - -The Python implementation of the service can be found inside -[pokemon_service.py](/rust-runtime/aws-smithy-http-python-server/examples/pokemon_service.py). - -## Build - -Since this example requires both the server and client SDK to be code-generated -from their [model](/codegen-server-test/model/pokemon.smithy), a Makefile is -provided to build and run the service. Just run `make build` to prepare the first -build. - -Once the example has been built successfully the first time, idiomatic `cargo` -can be used directly. - -`make distclean` can be used for a complete cleanup of all artefacts. - -### Uvloop - -The server can depend on [uvloop](https://pypi.org/project/uvloop/) for a -faster event loop implementation. Uvloop can be installed with your favourite -package manager or by using pip: - -```sh -pip instal uvloop -``` - -and it will be automatically used instead of the standard library event loop if -it is found in the dependencies' closure. - -### MacOs - -To compile and test on MacOs, please follow the official PyO3 guidelines on how -to [configure your linker](https://pyo3.rs/latest/building_and_distribution.html?highlight=rustflags#macos). - -Please note that the `.cargo/config.toml` with linkers override can be local to -your project. - -## Run - -`cargo run` can be used to start the Pokémon service on -`http://localhost:13734`. - -## Test - -`cargo test` can be used to spawn the Python service and run some simple integration -tests against it. - -More info can be found in the `tests` folder of `pokemon-service-test` package. diff --git a/rust-runtime/aws-smithy-http-server-python/src/tls.rs b/rust-runtime/aws-smithy-http-server-python/src/tls.rs index 538508fcec..c817cabaca 100644 --- a/rust-runtime/aws-smithy-http-server-python/src/tls.rs +++ b/rust-runtime/aws-smithy-http-server-python/src/tls.rs @@ -105,7 +105,7 @@ impl PyTlsConfig { #[pymethods] impl PyTlsConfig { #[new] - #[args(reload_secs = "86400")] // <- 1 Day by default + #[pyo3(signature = (key_path, cert_path, reload_secs=86400))] fn py_new(key_path: PathBuf, cert_path: PathBuf, reload_secs: u64) -> Self { // TODO(BugOnUpstream): `reload: &PyDelta` segfaults, create an issue on PyO3 Self { @@ -146,11 +146,11 @@ mod tests { const TEST_KEY: &str = concat!( env!("CARGO_MANIFEST_DIR"), - "/examples/pokemon-service-test/tests/testdata/localhost.key" + "/../../examples/python/pokemon-service-test/tests/testdata/localhost.key" ); const TEST_CERT: &str = concat!( env!("CARGO_MANIFEST_DIR"), - "/examples/pokemon-service-test/tests/testdata/localhost.crt" + "/../../examples/python/pokemon-service-test/tests/testdata/localhost.crt" ); #[test] diff --git a/rust-runtime/aws-smithy-http-server-python/src/tls/listener.rs b/rust-runtime/aws-smithy-http-server-python/src/tls/listener.rs index fb4f759f89..fb3aa7ac91 100644 --- a/rust-runtime/aws-smithy-http-server-python/src/tls/listener.rs +++ b/rust-runtime/aws-smithy-http-server-python/src/tls/listener.rs @@ -191,7 +191,7 @@ mod tests { assert!(response .unwrap_err() .to_string() - .contains("invalid peer certificate: InvalidCertValidity")); + .contains("invalid peer certificate: Expired")); } // Make a new acceptor with a valid cert and replace diff --git a/rust-runtime/aws-smithy-http-server-python/examples/stubgen.py b/rust-runtime/aws-smithy-http-server-python/stubgen.py similarity index 88% rename from rust-runtime/aws-smithy-http-server-python/examples/stubgen.py rename to rust-runtime/aws-smithy-http-server-python/stubgen.py index 30348838d2..458ee2b9c5 100644 --- a/rust-runtime/aws-smithy-http-server-python/examples/stubgen.py +++ b/rust-runtime/aws-smithy-http-server-python/stubgen.py @@ -2,11 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -import re + import inspect +import re import textwrap from pathlib import Path -from typing import Any, Set, Dict, List, Tuple, Optional +from typing import Any, Dict, List, Optional, Set, Tuple ROOT_MODULE_NAME_PLACEHOLDER = "__root_module_name__" @@ -36,11 +37,7 @@ def fix_path(self, path: str) -> str: Returns fixed version of given type path. It unescapes `\\[` and `\\]` and also populates placeholder for root module name. """ - return ( - path.replace(ROOT_MODULE_NAME_PLACEHOLDER, self.root_module_name) - .replace("\\[", "[") - .replace("\\]", "]") - ) + return path.replace(ROOT_MODULE_NAME_PLACEHOLDER, self.root_module_name).replace("\\[", "[").replace("\\]", "]") def submodule(self, path: Path) -> Writer: w = Writer(path, self.root_module_name) @@ -98,27 +95,21 @@ def __init__(self) -> None: def parse_type_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=1) if len(parts) != 2: - raise ValueError( - f"Invalid `:type` directive: `{line}` must be in `:type T:` format" - ) + raise ValueError(f"Invalid `:type` directive: `{line}` must be in `:type T:` format") res.types.append(parts[1].rstrip(":")) def parse_rtype_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=1) if len(parts) != 2: - raise ValueError( - f"Invalid `:rtype` directive: `{line}` must be in `:rtype T:` format" - ) + raise ValueError(f"Invalid `:rtype` directive: `{line}` must be in `:rtype T:` format") res.rtypes.append(parts[1].rstrip(":")) def parse_param_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=2) if len(parts) != 3: - raise ValueError( - f"Invalid `:param` directive: `{line}` must be in `:param name T:` format" - ) + raise ValueError(f"Invalid `:param` directive: `{line}` must be in `:param name T:` format") name = parts[1] ty = parts[2].rstrip(":") res.params.append((name, ty)) @@ -127,18 +118,14 @@ def parse_param_directive(line: str, res: DocstringParserResult): def parse_generic_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=1) if len(parts) != 2: - raise ValueError( - f"Invalid `:generic` directive: `{line}` must be in `:generic T:` format" - ) + raise ValueError(f"Invalid `:generic` directive: `{line}` must be in `:generic T:` format") res.generics.append(parts[1].rstrip(":")) def parse_extends_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=1) if len(parts) != 2: - raise ValueError( - f"Invalid `:extends` directive: `{line}` must be in `:extends Base[...]:` format" - ) + raise ValueError(f"Invalid `:extends` directive: `{line}` must be in `:extends Base[...]:` format") res.extends.append(parts[1].rstrip(":")) @@ -201,13 +188,13 @@ def clean_doc(obj: Any) -> str: if not doc: return "" - def predicate(l: str) -> bool: + def predicate(line: str) -> bool: for k in DocstringParserDirectives.keys(): - if l.startswith(f":{k} ") and l.endswith(":"): + if line.startswith(f":{k} ") and line.endswith(":"): return False return True - return "\n".join([l for l in doc.splitlines() if predicate(l)]).strip() + return "\n".join([line for line in doc.splitlines() if predicate(line)]).strip() def indent(code: str, level: int = 4) -> str: @@ -225,6 +212,10 @@ def is_fn_like(obj: Any) -> bool: ) +def is_scalar(obj: Any) -> bool: + return isinstance(obj, (str, float, int, bool)) + + def join(args: List[str], delim: str = "\n") -> str: return delim.join(filter(lambda x: x, args)) @@ -266,7 +257,7 @@ def make_function( sig: Optional[inspect.Signature] = None try: sig = inspect.signature(obj) - except: + except Exception: pass def has_default(param: str, ty: str) -> bool: @@ -312,9 +303,7 @@ def {name}({params}) -> {rtype}: def make_class(writer: Writer, name: str, klass: Any) -> str: - bases = list( - filter(lambda n: n != "object", map(lambda b: b.__name__, klass.__bases__)) - ) + bases = list(filter(lambda n: n != "object", map(lambda b: b.__name__, klass.__bases__))) class_sig = DocstringParser.parse_class(klass) if class_sig: (generics, extends) = class_sig @@ -386,7 +375,7 @@ class {name}{bases_str}: def walk_module(writer: Writer, mod: Any): exported = mod.__all__ - for (name, member) in inspect.getmembers(mod): + for name, member in inspect.getmembers(mod): if name not in exported: continue @@ -397,10 +386,25 @@ def walk_module(writer: Writer, mod: Any): writer.define(make_class(writer, name, member)) elif is_fn_like(member): writer.define(make_function(writer, name, member)) + elif is_scalar(member): + writer.define(f"{name}: {type(member).__name__} = ...") else: print(f"Unknown type: {member}") +def generate(module: str, outdir: str): + path = Path(outdir) / f"{module}.pyi" + writer = Writer( + path, + module, + ) + walk_module( + writer, + importlib.import_module(module), + ) + writer.dump() + + if __name__ == "__main__": import argparse import importlib @@ -410,13 +414,4 @@ def walk_module(writer: Writer, mod: Any): parser.add_argument("outdir") args = parser.parse_args() - path = Path(args.outdir) / f"{args.module}.pyi" - writer = Writer( - path, - args.module, - ) - walk_module( - writer, - importlib.import_module(args.module), - ) - writer.dump() + generate(args.module, args.outdir) diff --git a/rust-runtime/aws-smithy-http-server-python/stubgen.sh b/rust-runtime/aws-smithy-http-server-python/stubgen.sh new file mode 100755 index 0000000000..f7708845b3 --- /dev/null +++ b/rust-runtime/aws-smithy-http-server-python/stubgen.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -x + +if [ $# -lt 3 ]; then + echo "usage: $0 package manifest_path output_directory" + exit 1 +fi + +# input arguments +package=$1 +manifest=$2 +output=$3 + +# the directory of the script +source_dir="$(git rev-parse --show-toplevel)" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [ -n "$source_dir" ]; then + CARGO_TARGET_DIR="$source_dir/target" +else + CARGO_TARGET_DIR=$(mktemp -d) + mkdir -p "$CARGO_TARGET_DIR" + # cleanup temporary directory + function cleanup { + # shellcheck disable=2317 + rm -rf "$CARGO_TARGET_DIR" + } + # register the cleanup function to be called on the EXIT signal + trap cleanup EXIT +fi +export CARGO_TARGET_DIR + +shared_object_extension="so" +# generate the Python stubs, +if [ "$(uname)" == "Darwin" ]; then + shared_object_extension="dylib" + export CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-undefined -C link-arg=dynamic_lookup" + export CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-undefined -C link-arg=dynamic_lookup" +fi + +cargo build --manifest-path "$manifest" +# The link target have to end with .so to be sure it is importable by the stubgen.py script. +ln -sf "$CARGO_TARGET_DIR/debug/lib$package.$shared_object_extension" "$CARGO_TARGET_DIR/debug/$package.so" +PYTHONPATH=$CARGO_TARGET_DIR/debug:$PYTHONPATH python3 "$script_dir/stubgen.py" "$package" "$output" + +exit 0 diff --git a/rust-runtime/aws-smithy-http-server-python/examples/stubgen_test.py b/rust-runtime/aws-smithy-http-server-python/stubgen_test.py similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/stubgen_test.py rename to rust-runtime/aws-smithy-http-server-python/stubgen_test.py diff --git a/tools/ci-scripts/check-server-python-e2e-test b/tools/ci-scripts/check-server-python-e2e-test index fbaf9b0d67..26b9e15914 100755 --- a/tools/ci-scripts/check-server-python-e2e-test +++ b/tools/ci-scripts/check-server-python-e2e-test @@ -4,6 +4,6 @@ # SPDX-License-Identifier: Apache-2.0 set -eux -cd smithy-rs/rust-runtime/aws-smithy-http-server-python/examples +cd smithy-rs/examples/python make test clippy diff --git a/tools/ci-scripts/codegen-diff/diff_lib.py b/tools/ci-scripts/codegen-diff/diff_lib.py index 09f7edbf86..d39bbaba17 100644 --- a/tools/ci-scripts/codegen-diff/diff_lib.py +++ b/tools/ci-scripts/codegen-diff/diff_lib.py @@ -39,12 +39,12 @@ def checkout_commit_and_generate(revision_sha, branch_name, targets=None): def generate_and_commit_generated_code(revision_sha, targets=None): targets = targets or [ - target_codegen_client, - target_codegen_server, - target_aws_sdk, - target_codegen_server_python, - target_codegen_server_typescript - ] + target_codegen_client, + target_codegen_server, + target_aws_sdk, + target_codegen_server_python, + target_codegen_server_typescript + ] # Clean the build artifacts before continuing assemble_tasks = ' '.join([f'{t}:assemble' for t in targets]) clean_tasks = ' '.join([f'{t}:clean' for t in targets]) @@ -61,6 +61,7 @@ def generate_and_commit_generated_code(revision_sha, targets=None): if target in targets: get_cmd_output(f"mv {target}/build/smithyprojections/{target} {OUTPUT_PATH}/") if target == target_codegen_server: + get_cmd_output(f"./gradlew --rerun-tasks {target_codegen_server_python}:stubs") get_cmd_output(f"mv {target}/python/build/smithyprojections/{target}-python {OUTPUT_PATH}/") get_cmd_output(f"mv {target}/typescript/build/smithyprojections/{target}-typescript {OUTPUT_PATH}/")