diff --git a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt index cf2ff4b5d5..8de5cdd59c 100644 --- a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt +++ b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt @@ -469,6 +469,7 @@ class RustWriter private constructor( devDependenciesOnly = true, ) fileName == "package.json" -> rawWriter(fileName, debugMode = debugMode) + fileName == "stubgen.sh" -> rawWriter(fileName, debugMode = debugMode) else -> RustWriter(fileName, namespace, debugMode = debugMode) } } diff --git a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt index 7e53055195..6be653bd59 100644 --- a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt +++ b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt @@ -96,6 +96,8 @@ data class RuntimeConfig( val crateSrcPrefix: String = cratePrefix.replace("-", "_") + fun runtimeCratesPath(): String? = runtimeCrateLocation.path + fun smithyRuntimeCrate( runtimeCrateName: String, optional: Boolean = false, diff --git a/codegen-server-test/python/build.gradle.kts b/codegen-server-test/python/build.gradle.kts index 87ff7cc5bd..e70e2b0e5d 100644 --- a/codegen-server-test/python/build.gradle.kts +++ b/codegen-server-test/python/build.gradle.kts @@ -66,7 +66,7 @@ val allCodegenTests = "../../codegen-core/common-test-models".let { commonModels "rest_json_extras", imports = listOf("$commonModels/rest-json-extras.smithy"), ), - // TODO(https://github.com/awslabs/smithy-rs/issues/2551) + // TODO(https://github.com/awslabs/smithy-rs/issues/2477) // CodegenTest( // "aws.protocoltests.restjson.validation#RestJsonValidation", // "rest_json_validation", @@ -104,6 +104,21 @@ project.registerGenerateSmithyBuildTask(rootProject, pluginName, allCodegenTests project.registerGenerateCargoWorkspaceTask(rootProject, pluginName, allCodegenTests, workingDirUnderBuildDir) project.registerGenerateCargoConfigTomlTask(buildDir.resolve(workingDirUnderBuildDir)) +tasks.register("stubs") { + description = "Generate Python stubs for all models" + dependsOn("assemble") + + doLast { + allCodegenTests.forEach { test -> + val crateDir = "$buildDir/$workingDirUnderBuildDir/${test.module}/$pluginName" + val moduleName = test.module.replace("-", "_") + exec { + commandLine("bash", "$crateDir/stubgen.sh", moduleName, "$crateDir/Cargo.toml", "$crateDir/python/$moduleName") + } + } + } +} + tasks["smithyBuildJar"].dependsOn("generateSmithyBuild") tasks["assemble"].finalizedBy("generateCargoWorkspace") diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt index 911edb893e..2782e44b94 100644 --- a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt @@ -15,8 +15,8 @@ import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig * For a dependency that is used in the client, or in both the client and the server, use [CargoDependency] directly. */ object PythonServerCargoDependency { - val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.17")) - val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.17"), features = setOf("attributes", "tokio-runtime")) + val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.18")) + val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.18"), features = setOf("attributes", "tokio-runtime")) val Tokio: CargoDependency = CargoDependency("tokio", CratesIo("1.20.1"), features = setOf("full")) val TokioStream: CargoDependency = CargoDependency("tokio-stream", CratesIo("0.1.12")) val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1")) diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt index 9e353db2cd..ff4b6be482 100644 --- a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt @@ -23,6 +23,7 @@ import software.amazon.smithy.rust.codegen.server.python.smithy.generators.Pytho import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenContext import software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOperationsDecorator import software.amazon.smithy.rust.codegen.server.smithy.customize.ServerCodegenDecorator +import java.io.File /** * Configure the [lib] section of `Cargo.toml`. @@ -194,6 +195,31 @@ class PyTypedMarkerDecorator : ServerCodegenDecorator { } } +/** + * Copies the stubgen scripts to the generated crate root. + * + * The shell script `stubgen.sh` runs a quick build and uses `stubgen.py` to generate mypy compatibile + * types stubs for the project. + */ +class AddStubgenScriptDecorator : ServerCodegenDecorator { + override val name: String = "AddStubgenScriptDecorator" + override val order: Byte = 0 + + override fun extras(codegenContext: ServerCodegenContext, rustCrate: RustCrate) { + val runtimeCratesPath = codegenContext.runtimeConfig.runtimeCratesPath() + val stubgenPythonLocation = "$runtimeCratesPath/aws-smithy-http-server-python/stubgen.py" + val stubgenPythonContent = File(stubgenPythonLocation).readText(Charsets.UTF_8) + rustCrate.withFile("stubgen.py") { + writeWithNoFormatting("$stubgenPythonContent") + } + val stubgenShellLocation = "$runtimeCratesPath/aws-smithy-http-server-python/stubgen.sh" + val stubgenShellContent = File(stubgenShellLocation).readText(Charsets.UTF_8) + rustCrate.withFile("stubgen.sh") { + writeWithNoFormatting("$stubgenShellContent") + } + } +} + val DECORATORS = arrayOf( /** * Add the [InternalServerError] error to all operations. @@ -214,4 +240,6 @@ val DECORATORS = arrayOf( InitPyDecorator(), // Generate `py.typed` for the Python source. PyTypedMarkerDecorator(), + // Generate scripts for stub generation. + AddStubgenScriptDecorator(), ) diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt index 5baf6e83ea..577999a724 100644 --- a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt @@ -11,7 +11,9 @@ import software.amazon.smithy.model.shapes.ResourceShape import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.shapes.Shape import software.amazon.smithy.model.shapes.UnionShape +import software.amazon.smithy.rust.codegen.core.Version import software.amazon.smithy.rust.codegen.core.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.core.rustlang.rust import software.amazon.smithy.rust.codegen.core.rustlang.rustBlockTemplate import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate import software.amazon.smithy.rust.codegen.core.smithy.RustCrate @@ -51,6 +53,8 @@ class PythonServerModuleGenerator( renderPyTlsTypes() renderPyLambdaTypes() renderPyApplicationType() + renderCodegenVersion() + rust("Ok(())") } } } @@ -62,13 +66,23 @@ class PythonServerModuleGenerator( let input = #{pyo3}::types::PyModule::new(py, "input")?; let output = #{pyo3}::types::PyModule::new(py, "output")?; let error = #{pyo3}::types::PyModule::new(py, "error")?; - let model = #{pyo3}::types::PyModule::new(py, "model")?; """, *codegenScope, ) + // The `model` type section can be unused in models like `simple`, so we accommodate for it. + var visitedModelType = false serviceShapes.forEach { shape -> val moduleType = moduleType(shape) if (moduleType != null) { + if (moduleType == "model" && !visitedModelType) { + rustTemplate( + """ + let model = #{pyo3}::types::PyModule::new(py, "model")?; + """, + *codegenScope, + ) + visitedModelType = true + } when (shape) { is UnionShape -> rustTemplate( """ @@ -93,11 +107,18 @@ class PythonServerModuleGenerator( m.add_submodule(output)?; #{pyo3}::py_run!(py, error, "import sys; sys.modules['$libName.error'] = error"); m.add_submodule(error)?; - #{pyo3}::py_run!(py, model, "import sys; sys.modules['$libName.model'] = model"); - m.add_submodule(model)?; """, *codegenScope, ) + if (visitedModelType) { + rustTemplate( + """ + #{pyo3}::py_run!(py, model, "import sys; sys.modules['$libName.model'] = model"); + m.add_submodule(model)?; + """, + *codegenScope, + ) + } } // Render wrapper types that are substituted to the ones coming from `aws_smithy_types`. @@ -211,13 +232,12 @@ class PythonServerModuleGenerator( // Render Python application type. private fun RustWriter.renderPyApplicationType() { - rustTemplate( - """ - m.add_class::()?; - Ok(()) - """, - *codegenScope, - ) + rust("""m.add_class::()?;""") + } + + // Render the codegeneration version as module attribute. + private fun RustWriter.renderCodegenVersion() { + rust("""m.add("CODEGEN_VERSION", "${Version.crateVersion()}")?;""") } // Convert to symbol and check the namespace to figure out where they should be imported from. diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCrateInlineModuleComposingWriter.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCrateInlineModuleComposingWriter.kt index 32abe41d69..96b3b5739c 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCrateInlineModuleComposingWriter.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCrateInlineModuleComposingWriter.kt @@ -335,7 +335,9 @@ class InnerModule(private val moduleDocProvider: ModuleDocProvider, debugMode: B inlineWriter } else { check(inlineModuleAndWriter.inlineModule == lookForModule) { - "The two inline modules have the same name but different attributes on them." + """The two inline modules have the same name but different attributes on them: + 1) ${inlineModuleAndWriter.inlineModule} + 2) $lookForModule""" } inlineModuleAndWriter.writer diff --git a/rust-runtime/aws-smithy-http-server-python/examples/.gitignore b/examples/python/.gitignore similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/.gitignore rename to examples/python/.gitignore diff --git a/rust-runtime/aws-smithy-http-server-python/examples/Cargo.toml b/examples/python/Cargo.toml similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/Cargo.toml rename to examples/python/Cargo.toml diff --git a/rust-runtime/aws-smithy-http-server-python/examples/Makefile b/examples/python/Makefile similarity index 90% rename from rust-runtime/aws-smithy-http-server-python/examples/Makefile rename to examples/python/Makefile index 3b73f08ca6..56213fe289 100644 --- a/rust-runtime/aws-smithy-http-server-python/examples/Makefile +++ b/examples/python/Makefile @@ -35,25 +35,23 @@ install-wheel: find $(WHEELS) -type f -name '*.whl' | xargs python3 -m pip install --user --force-reinstall generate-stubs: - python3 $(CUR_DIR)/stubgen.py pokemon_service_server_sdk $(SERVER_SDK_DST)/python/pokemon_service_server_sdk + bash $(SERVER_SDK_DST)/stubgen.sh pokemon_service_server_sdk $(SERVER_SDK_DST)/Cargo.toml $(SERVER_SDK_DST)/python/pokemon_service_server_sdk build: codegen + $(MAKE) generate-stubs $(MAKE) build-wheel - $(MAKE) install-wheel + +release: codegen $(MAKE) generate-stubs $(MAKE) build-wheel-release - $(MAKE) install-wheel run: build python3 $(CUR_DIR)/pokemon_service.py -py-check: build +py-check: install-wheel python3 -m mypy pokemon_service.py -py-test: - python3 stubgen_test.py - -test: build py-check py-test +test: build py-check cargo test clippy: codegen diff --git a/examples/python/README.md b/examples/python/README.md new file mode 100644 index 0000000000..ebe6cfb39e --- /dev/null +++ b/examples/python/README.md @@ -0,0 +1,94 @@ +# Smithy Rust/Python Server SDK example + +This folder contains an example service called Pokémon Service used to showcase +the service framework Python bindings capabilities and to run benchmarks. + +The Python implementation of the service can be found inside +[pokemon_service.py](./pokemon_service.py). + +* [Build](#build) + * [Build dependencies](#build-dependencies) + * [Makefile](#makefile) + * [Python stub generation](#python-stub-generation) +* [Run](#run) +* [Test](#test) +* [Uvloop](#uvloop) +* [MacOs](#macos) + +## Build + +Since this example requires both the server and client SDK to be code-generated +from their [model](/codegen-server-test/model/pokemon.smithy), a Makefile is +provided to build and run the service. Just run `make build` to prepare the first +build. + +### Build dependencies + +Ensure these dependencies are installed. + +``` +pip install maturin uvloop aiohttp mypy +``` + +### Makefile + +The build logic is drive by the Makefile: + +* `make codegen`: run the codegenerator. +* `make build`: build the Maturin package in debug mode (includes type stubs + generation). +* `make release`: build the Maturin package in release mode (includes type stub + generation). +* `make install`: install the latest release or debug build. +* `make run`: run the example server. +* `make test`: run the end-to-end integration test. + +### Python stub generation + +We support the generation of mypy python stubs and every SDK crate ships with +a script called `stubgen.sh`. **Note that the script is not called +automatically as part of the build**. We suggest users to call it after code generation. +It will do first compilation of the crate, generate the types and exit. + +The script takes some command line arguments: + +``` +./stubgen.sh module_name manifest_path output_directory +``` + +* module_name: name of the Python module to generate stubs for, IE `pokemon_service_server_sdk`. +* manifest_path: path for the crate manifest used to build the types. +* output_directory: directory where to generate the stub hierarchy. **This + directory should be a folder `python/$module_name` in the root of the Maturin package.** + +## Run + +`make run` can be used to start the Pokémon service on `http://localhost:13734`. + +## Test + +`make test` can be used to spawn the Python service and run some simple integration +tests against it. + +More info can be found in the `tests` folder of `pokemon-service-test` package. + +## Uvloop + +The server can depend on [uvloop](https://pypi.org/project/uvloop/) for a +faster event loop implementation. Uvloop can be installed with your favourite +package manager or by using pip: + +```sh +pip instal uvloop +``` + +and it will be automatically used instead of the standard library event loop if +it is found in the dependencies' closure. + +## MacOs + +To compile and test on MacOs, please follow the official PyO3 guidelines on how +to [configure your linker](https://pyo3.rs/latest/building_and_distribution.html?highlight=rustflags#macos). + +Please note that the `.cargo/config.toml` with linkers override can be local to +your project. diff --git a/rust-runtime/aws-smithy-http-server-python/examples/mypy.ini b/examples/python/mypy.ini similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/mypy.ini rename to examples/python/mypy.ini diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/Cargo.toml b/examples/python/pokemon-service-test/Cargo.toml similarity index 53% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/Cargo.toml rename to examples/python/pokemon-service-test/Cargo.toml index 6edcebf160..8dbe8c6974 100644 --- a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/Cargo.toml +++ b/examples/python/pokemon-service-test/Cargo.toml @@ -9,15 +9,15 @@ description = "Run tests against the Python server implementation" [dev-dependencies] rand = "0.8" async-stream = "0.3" -command-group = "1.0" +command-group = "2.1.0" tokio = { version = "1.20.1", features = ["full"] } -serial_test = "0.9.0" +serial_test = "2.0.0" rustls-pemfile = "1.0.1" -tokio-rustls = "0.23.4" -hyper-rustls = { version = "0.23.0", features = ["http2"] } +tokio-rustls = "0.24.0" +hyper-rustls = { version = "0.24.0", features = ["http2"] } # Local paths -aws-smithy-client = { path = "../../../aws-smithy-client/", features = ["rustls"] } -aws-smithy-http = { path = "../../../aws-smithy-http/" } -aws-smithy-types = { path = "../../../aws-smithy-types/" } +aws-smithy-client = { path = "../../../rust-runtime/aws-smithy-client/", features = ["rustls"] } +aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http/" } +aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types/" } pokemon-service-client = { path = "../pokemon-service-client/" } diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/helpers.rs b/examples/python/pokemon-service-test/tests/helpers.rs similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/helpers.rs rename to examples/python/pokemon-service-test/tests/helpers.rs diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/simple_integration_test.rs b/examples/python/pokemon-service-test/tests/simple_integration_test.rs similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/simple_integration_test.rs rename to examples/python/pokemon-service-test/tests/simple_integration_test.rs diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/testdata/localhost.crt b/examples/python/pokemon-service-test/tests/testdata/localhost.crt similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/testdata/localhost.crt rename to examples/python/pokemon-service-test/tests/testdata/localhost.crt diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/testdata/localhost.key b/examples/python/pokemon-service-test/tests/testdata/localhost.key similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon-service-test/tests/testdata/localhost.key rename to examples/python/pokemon-service-test/tests/testdata/localhost.key diff --git a/rust-runtime/aws-smithy-http-server-python/examples/pokemon_service.py b/examples/python/pokemon_service.py similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/pokemon_service.py rename to examples/python/pokemon_service.py diff --git a/rust-runtime/aws-smithy-http-server-python/Cargo.toml b/rust-runtime/aws-smithy-http-server-python/Cargo.toml index 69e9577160..738b86418b 100644 --- a/rust-runtime/aws-smithy-http-server-python/Cargo.toml +++ b/rust-runtime/aws-smithy-http-server-python/Cargo.toml @@ -22,9 +22,9 @@ bytes = "1.2" futures = "0.3" http = "0.2" hyper = { version = "0.14.20", features = ["server", "http1", "http2", "tcp", "stream"] } -tls-listener = { version = "0.5.1", features = ["rustls", "hyper-h2"] } +tls-listener = { version = "0.7.0", features = ["rustls", "hyper-h2"] } rustls-pemfile = "1.0.1" -tokio-rustls = "0.23.4" +tokio-rustls = "0.24.0" lambda_http = { version = "0.7.1" } # There is a breaking change in `lambda_runtime` between `0.7.0` and `0.7.1`, # and `lambda_http` depends on `0.7` which by default resolves to `0.7.1` but in our CI @@ -34,10 +34,10 @@ lambda_runtime = { version = "0.7.1" } num_cpus = "1.13.1" parking_lot = "0.12.1" pin-project-lite = "0.2" -pyo3 = "0.17.0" -pyo3-asyncio = { version = "0.17.0", features = ["tokio-runtime"] } +pyo3 = "0.18.2" +pyo3-asyncio = { version = "0.18.0", features = ["tokio-runtime"] } signal-hook = { version = "0.3.14", features = ["extended-siginfo"] } -socket2 = { version = "0.4.4", features = ["all"] } +socket2 = { version = "0.5.2", features = ["all"] } thiserror = "1.0.32" tokio = { version = "1.20.1", features = ["full"] } tokio-stream = "0.1" @@ -51,9 +51,9 @@ pretty_assertions = "1" futures-util = { version = "0.3.16", default-features = false } tower-test = "0.4" tokio-test = "0.4" -pyo3-asyncio = { version = "0.17.0", features = ["testing", "attributes", "tokio-runtime", "unstable-streams"] } +pyo3-asyncio = { version = "0.18.0", features = ["testing", "attributes", "tokio-runtime", "unstable-streams"] } rcgen = "0.10.0" -hyper-rustls = { version = "0.23.1", features = ["http2"] } +hyper-rustls = { version = "0.24.0", features = ["http2"] } # PyO3 Asyncio tests cannot use Cargo's default testing harness because `asyncio` # wants to control the main thread. So we need to use testing harness provided by `pyo3_asyncio` diff --git a/rust-runtime/aws-smithy-http-server-python/examples/README.md b/rust-runtime/aws-smithy-http-server-python/examples/README.md deleted file mode 100644 index f765f4b57b..0000000000 --- a/rust-runtime/aws-smithy-http-server-python/examples/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Smithy Rust/Python Server SDK example - -This folder contains an example service called Pokémon Service used to showcase -the service framework Python bindings capabilities and to run benchmarks. - -The Python implementation of the service can be found inside -[pokemon_service.py](/rust-runtime/aws-smithy-http-python-server/examples/pokemon_service.py). - -## Build - -Since this example requires both the server and client SDK to be code-generated -from their [model](/codegen-server-test/model/pokemon.smithy), a Makefile is -provided to build and run the service. Just run `make build` to prepare the first -build. - -Once the example has been built successfully the first time, idiomatic `cargo` -can be used directly. - -`make distclean` can be used for a complete cleanup of all artefacts. - -### Uvloop - -The server can depend on [uvloop](https://pypi.org/project/uvloop/) for a -faster event loop implementation. Uvloop can be installed with your favourite -package manager or by using pip: - -```sh -pip instal uvloop -``` - -and it will be automatically used instead of the standard library event loop if -it is found in the dependencies' closure. - -### MacOs - -To compile and test on MacOs, please follow the official PyO3 guidelines on how -to [configure your linker](https://pyo3.rs/latest/building_and_distribution.html?highlight=rustflags#macos). - -Please note that the `.cargo/config.toml` with linkers override can be local to -your project. - -## Run - -`cargo run` can be used to start the Pokémon service on -`http://localhost:13734`. - -## Test - -`cargo test` can be used to spawn the Python service and run some simple integration -tests against it. - -More info can be found in the `tests` folder of `pokemon-service-test` package. diff --git a/rust-runtime/aws-smithy-http-server-python/src/tls.rs b/rust-runtime/aws-smithy-http-server-python/src/tls.rs index 538508fcec..c817cabaca 100644 --- a/rust-runtime/aws-smithy-http-server-python/src/tls.rs +++ b/rust-runtime/aws-smithy-http-server-python/src/tls.rs @@ -105,7 +105,7 @@ impl PyTlsConfig { #[pymethods] impl PyTlsConfig { #[new] - #[args(reload_secs = "86400")] // <- 1 Day by default + #[pyo3(signature = (key_path, cert_path, reload_secs=86400))] fn py_new(key_path: PathBuf, cert_path: PathBuf, reload_secs: u64) -> Self { // TODO(BugOnUpstream): `reload: &PyDelta` segfaults, create an issue on PyO3 Self { @@ -146,11 +146,11 @@ mod tests { const TEST_KEY: &str = concat!( env!("CARGO_MANIFEST_DIR"), - "/examples/pokemon-service-test/tests/testdata/localhost.key" + "/../../examples/python/pokemon-service-test/tests/testdata/localhost.key" ); const TEST_CERT: &str = concat!( env!("CARGO_MANIFEST_DIR"), - "/examples/pokemon-service-test/tests/testdata/localhost.crt" + "/../../examples/python/pokemon-service-test/tests/testdata/localhost.crt" ); #[test] diff --git a/rust-runtime/aws-smithy-http-server-python/src/tls/listener.rs b/rust-runtime/aws-smithy-http-server-python/src/tls/listener.rs index fb4f759f89..fb3aa7ac91 100644 --- a/rust-runtime/aws-smithy-http-server-python/src/tls/listener.rs +++ b/rust-runtime/aws-smithy-http-server-python/src/tls/listener.rs @@ -191,7 +191,7 @@ mod tests { assert!(response .unwrap_err() .to_string() - .contains("invalid peer certificate: InvalidCertValidity")); + .contains("invalid peer certificate: Expired")); } // Make a new acceptor with a valid cert and replace diff --git a/rust-runtime/aws-smithy-http-server-python/examples/stubgen.py b/rust-runtime/aws-smithy-http-server-python/stubgen.py similarity index 88% rename from rust-runtime/aws-smithy-http-server-python/examples/stubgen.py rename to rust-runtime/aws-smithy-http-server-python/stubgen.py index 30348838d2..458ee2b9c5 100644 --- a/rust-runtime/aws-smithy-http-server-python/examples/stubgen.py +++ b/rust-runtime/aws-smithy-http-server-python/stubgen.py @@ -2,11 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -import re + import inspect +import re import textwrap from pathlib import Path -from typing import Any, Set, Dict, List, Tuple, Optional +from typing import Any, Dict, List, Optional, Set, Tuple ROOT_MODULE_NAME_PLACEHOLDER = "__root_module_name__" @@ -36,11 +37,7 @@ def fix_path(self, path: str) -> str: Returns fixed version of given type path. It unescapes `\\[` and `\\]` and also populates placeholder for root module name. """ - return ( - path.replace(ROOT_MODULE_NAME_PLACEHOLDER, self.root_module_name) - .replace("\\[", "[") - .replace("\\]", "]") - ) + return path.replace(ROOT_MODULE_NAME_PLACEHOLDER, self.root_module_name).replace("\\[", "[").replace("\\]", "]") def submodule(self, path: Path) -> Writer: w = Writer(path, self.root_module_name) @@ -98,27 +95,21 @@ def __init__(self) -> None: def parse_type_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=1) if len(parts) != 2: - raise ValueError( - f"Invalid `:type` directive: `{line}` must be in `:type T:` format" - ) + raise ValueError(f"Invalid `:type` directive: `{line}` must be in `:type T:` format") res.types.append(parts[1].rstrip(":")) def parse_rtype_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=1) if len(parts) != 2: - raise ValueError( - f"Invalid `:rtype` directive: `{line}` must be in `:rtype T:` format" - ) + raise ValueError(f"Invalid `:rtype` directive: `{line}` must be in `:rtype T:` format") res.rtypes.append(parts[1].rstrip(":")) def parse_param_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=2) if len(parts) != 3: - raise ValueError( - f"Invalid `:param` directive: `{line}` must be in `:param name T:` format" - ) + raise ValueError(f"Invalid `:param` directive: `{line}` must be in `:param name T:` format") name = parts[1] ty = parts[2].rstrip(":") res.params.append((name, ty)) @@ -127,18 +118,14 @@ def parse_param_directive(line: str, res: DocstringParserResult): def parse_generic_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=1) if len(parts) != 2: - raise ValueError( - f"Invalid `:generic` directive: `{line}` must be in `:generic T:` format" - ) + raise ValueError(f"Invalid `:generic` directive: `{line}` must be in `:generic T:` format") res.generics.append(parts[1].rstrip(":")) def parse_extends_directive(line: str, res: DocstringParserResult): parts = line.split(" ", maxsplit=1) if len(parts) != 2: - raise ValueError( - f"Invalid `:extends` directive: `{line}` must be in `:extends Base[...]:` format" - ) + raise ValueError(f"Invalid `:extends` directive: `{line}` must be in `:extends Base[...]:` format") res.extends.append(parts[1].rstrip(":")) @@ -201,13 +188,13 @@ def clean_doc(obj: Any) -> str: if not doc: return "" - def predicate(l: str) -> bool: + def predicate(line: str) -> bool: for k in DocstringParserDirectives.keys(): - if l.startswith(f":{k} ") and l.endswith(":"): + if line.startswith(f":{k} ") and line.endswith(":"): return False return True - return "\n".join([l for l in doc.splitlines() if predicate(l)]).strip() + return "\n".join([line for line in doc.splitlines() if predicate(line)]).strip() def indent(code: str, level: int = 4) -> str: @@ -225,6 +212,10 @@ def is_fn_like(obj: Any) -> bool: ) +def is_scalar(obj: Any) -> bool: + return isinstance(obj, (str, float, int, bool)) + + def join(args: List[str], delim: str = "\n") -> str: return delim.join(filter(lambda x: x, args)) @@ -266,7 +257,7 @@ def make_function( sig: Optional[inspect.Signature] = None try: sig = inspect.signature(obj) - except: + except Exception: pass def has_default(param: str, ty: str) -> bool: @@ -312,9 +303,7 @@ def {name}({params}) -> {rtype}: def make_class(writer: Writer, name: str, klass: Any) -> str: - bases = list( - filter(lambda n: n != "object", map(lambda b: b.__name__, klass.__bases__)) - ) + bases = list(filter(lambda n: n != "object", map(lambda b: b.__name__, klass.__bases__))) class_sig = DocstringParser.parse_class(klass) if class_sig: (generics, extends) = class_sig @@ -386,7 +375,7 @@ class {name}{bases_str}: def walk_module(writer: Writer, mod: Any): exported = mod.__all__ - for (name, member) in inspect.getmembers(mod): + for name, member in inspect.getmembers(mod): if name not in exported: continue @@ -397,10 +386,25 @@ def walk_module(writer: Writer, mod: Any): writer.define(make_class(writer, name, member)) elif is_fn_like(member): writer.define(make_function(writer, name, member)) + elif is_scalar(member): + writer.define(f"{name}: {type(member).__name__} = ...") else: print(f"Unknown type: {member}") +def generate(module: str, outdir: str): + path = Path(outdir) / f"{module}.pyi" + writer = Writer( + path, + module, + ) + walk_module( + writer, + importlib.import_module(module), + ) + writer.dump() + + if __name__ == "__main__": import argparse import importlib @@ -410,13 +414,4 @@ def walk_module(writer: Writer, mod: Any): parser.add_argument("outdir") args = parser.parse_args() - path = Path(args.outdir) / f"{args.module}.pyi" - writer = Writer( - path, - args.module, - ) - walk_module( - writer, - importlib.import_module(args.module), - ) - writer.dump() + generate(args.module, args.outdir) diff --git a/rust-runtime/aws-smithy-http-server-python/stubgen.sh b/rust-runtime/aws-smithy-http-server-python/stubgen.sh new file mode 100755 index 0000000000..f7708845b3 --- /dev/null +++ b/rust-runtime/aws-smithy-http-server-python/stubgen.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -x + +if [ $# -lt 3 ]; then + echo "usage: $0 package manifest_path output_directory" + exit 1 +fi + +# input arguments +package=$1 +manifest=$2 +output=$3 + +# the directory of the script +source_dir="$(git rev-parse --show-toplevel)" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [ -n "$source_dir" ]; then + CARGO_TARGET_DIR="$source_dir/target" +else + CARGO_TARGET_DIR=$(mktemp -d) + mkdir -p "$CARGO_TARGET_DIR" + # cleanup temporary directory + function cleanup { + # shellcheck disable=2317 + rm -rf "$CARGO_TARGET_DIR" + } + # register the cleanup function to be called on the EXIT signal + trap cleanup EXIT +fi +export CARGO_TARGET_DIR + +shared_object_extension="so" +# generate the Python stubs, +if [ "$(uname)" == "Darwin" ]; then + shared_object_extension="dylib" + export CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-undefined -C link-arg=dynamic_lookup" + export CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-undefined -C link-arg=dynamic_lookup" +fi + +cargo build --manifest-path "$manifest" +# The link target have to end with .so to be sure it is importable by the stubgen.py script. +ln -sf "$CARGO_TARGET_DIR/debug/lib$package.$shared_object_extension" "$CARGO_TARGET_DIR/debug/$package.so" +PYTHONPATH=$CARGO_TARGET_DIR/debug:$PYTHONPATH python3 "$script_dir/stubgen.py" "$package" "$output" + +exit 0 diff --git a/rust-runtime/aws-smithy-http-server-python/examples/stubgen_test.py b/rust-runtime/aws-smithy-http-server-python/stubgen_test.py similarity index 100% rename from rust-runtime/aws-smithy-http-server-python/examples/stubgen_test.py rename to rust-runtime/aws-smithy-http-server-python/stubgen_test.py diff --git a/tools/ci-scripts/check-server-python-e2e-test b/tools/ci-scripts/check-server-python-e2e-test index fbaf9b0d67..26b9e15914 100755 --- a/tools/ci-scripts/check-server-python-e2e-test +++ b/tools/ci-scripts/check-server-python-e2e-test @@ -4,6 +4,6 @@ # SPDX-License-Identifier: Apache-2.0 set -eux -cd smithy-rs/rust-runtime/aws-smithy-http-server-python/examples +cd smithy-rs/examples/python make test clippy diff --git a/tools/ci-scripts/codegen-diff/diff_lib.py b/tools/ci-scripts/codegen-diff/diff_lib.py index 09f7edbf86..d39bbaba17 100644 --- a/tools/ci-scripts/codegen-diff/diff_lib.py +++ b/tools/ci-scripts/codegen-diff/diff_lib.py @@ -39,12 +39,12 @@ def checkout_commit_and_generate(revision_sha, branch_name, targets=None): def generate_and_commit_generated_code(revision_sha, targets=None): targets = targets or [ - target_codegen_client, - target_codegen_server, - target_aws_sdk, - target_codegen_server_python, - target_codegen_server_typescript - ] + target_codegen_client, + target_codegen_server, + target_aws_sdk, + target_codegen_server_python, + target_codegen_server_typescript + ] # Clean the build artifacts before continuing assemble_tasks = ' '.join([f'{t}:assemble' for t in targets]) clean_tasks = ' '.join([f'{t}:clean' for t in targets]) @@ -61,6 +61,7 @@ def generate_and_commit_generated_code(revision_sha, targets=None): if target in targets: get_cmd_output(f"mv {target}/build/smithyprojections/{target} {OUTPUT_PATH}/") if target == target_codegen_server: + get_cmd_output(f"./gradlew --rerun-tasks {target_codegen_server_python}:stubs") get_cmd_output(f"mv {target}/python/build/smithyprojections/{target}-python {OUTPUT_PATH}/") get_cmd_output(f"mv {target}/typescript/build/smithyprojections/{target}-typescript {OUTPUT_PATH}/")