Python: Use Maturin to build SDKs (#2025)

* Remove `lib` prefix from generated module names

* Build Pokemon service with `Maturin`

* Fix Maturin build on CI

* Generate minimal `pyproject.toml` for generated SDKs to build from source using Maturin

* Fix `ktlint` issues

* Bring back type stubs for Pokemon service

* Update instructions for Lambda

* Make `build-wheel` and `build-wheel-release` to depend on `codegen`

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt

@@ -5,6 +5,7 @@
 
 package software.amazon.smithy.rust.codegen.server.python.smithy.customizations
 
+import com.moandjiezana.toml.TomlWriter
 import software.amazon.smithy.model.neighbor.Walker
 import software.amazon.smithy.rust.codegen.client.smithy.customize.RustCodegenDecorator
 import software.amazon.smithy.rust.codegen.core.rustlang.Writable
@@ -103,6 +104,30 @@ class PubUsePythonTypesDecorator : RustCodegenDecorator<ServerProtocolGenerator,
         clazz.isAssignableFrom(ServerCodegenContext::class.java)
 }
 
+/**
+ * Generates `pyproject.toml` for the crate.
+ *  - Configures Maturin as the build system
+ */
+class PyProjectTomlDecorator : RustCodegenDecorator<ServerProtocolGenerator, ServerCodegenContext> {
+    override val name: String = "PyProjectTomlDecorator"
+    override val order: Byte = 0
+
+    override fun extras(codegenContext: ServerCodegenContext, rustCrate: RustCrate) {
+        rustCrate.withFile("pyproject.toml") {
+            val config = mapOf(
+                "build-system" to listOfNotNull(
+                    "requires" to listOfNotNull("maturin>=0.14,<0.15"),
+                    "build-backend" to "maturin",
+                ).toMap(),
+            )
+            writeWithNoFormatting(TomlWriter().write(config))
+        }
+    }
+
+    override fun supportsCodegenContext(clazz: Class<out CodegenContext>): Boolean =
+        clazz.isAssignableFrom(ServerCodegenContext::class.java)
+}
+
 val DECORATORS = listOf(
     /**
      * Add the [InternalServerError] error to all operations.
@@ -115,4 +140,6 @@ val DECORATORS = listOf(
     PubUsePythonTypesDecorator(),
     // Render the Python shared library export.
     PythonExportModuleDecorator(),
+    // Generate `pyproject.toml` for the crate.
+    PyProjectTomlDecorator(),
 )

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonApplicationGenerator.kt

@@ -67,7 +67,7 @@ class PythonApplicationGenerator(
     private val operations: List<OperationShape>,
 ) {
     private val symbolProvider = codegenContext.symbolProvider
-    private val libName = "lib${codegenContext.settings.moduleName.toSnakeCase()}"
+    private val libName = codegenContext.settings.moduleName.toSnakeCase()
     private val runtimeConfig = codegenContext.runtimeConfig
     private val service = codegenContext.serviceShape
     private val serviceName = service.id.name.toPascalCase()
@@ -278,7 +278,6 @@ class PythonApplicationGenerator(
                     self.run_server(py, address, port, backlog, workers, tls)
                 }
                 /// Lambda entrypoint: start the server on Lambda.
-                ##[cfg(feature = "aws-lambda")]
                 ##[pyo3(text_signature = "(${'$'}self)")]
                 pub fn run_lambda(
                     &mut self,

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt

@@ -29,7 +29,7 @@ class PythonServerModuleGenerator(
         "pyo3" to PythonServerCargoDependency.PyO3.toType(),
     )
     private val symbolProvider = codegenContext.symbolProvider
-    private val libName = "lib${codegenContext.settings.moduleName.toSnakeCase()}"
+    private val libName = codegenContext.settings.moduleName.toSnakeCase()
 
     fun render() {
         rustCrate.withModule(

rust-runtime/aws-smithy-http-server-python/README.md

@@ -13,8 +13,8 @@ instead of the [Hyper](https://hyper.rs/) HTTP server.
 In your `app.py`:
 
 ```diff
-from libpokemon_service_server_sdk import App
-from libpokemon_service_server_sdk.error import ResourceNotFoundException
+from pokemon_service_server_sdk import App
+from pokemon_service_server_sdk.error import ResourceNotFoundException
 
 # ...
 
@@ -44,19 +44,19 @@ FROM public.ecr.aws/lambda/python:3.8-x86_64
 
 # Copy your application code to `LAMBDA_TASK_ROOT`
 COPY app.py ${LAMBDA_TASK_ROOT}
-# When you build your Server SDK for your service you get a shared library
-# that is importable in Python. You need to copy that shared library to same folder
-# with your application code, so it can be imported by your application.
-# Note that you need to build your library for Linux,
-# if you are on a different platform you can consult to
+
+# When you build your Server SDK for your service, you will get a Python wheel.
+# You just need to copy that wheel and install it via `pip` inside your image.
+# Note that you need to build your library for Linux, and Python version used to
+# build your SDK should match with your image's Python version.
+# For cross compiling, you can consult to:
 # https://pyo3.rs/latest/building_and_distribution.html#cross-compiling
-# for cross compiling.
-COPY lib_pokemon_service_server_sdk.so ${LAMBDA_TASK_ROOT}
+COPY wheels/ ${LAMBDA_TASK_ROOT}/wheels
+RUN pip3 install ${LAMBDA_TASK_ROOT}/wheels/*.whl
 
-# You can install your application's dependencies using file `requirements.txt`
-# from your project folder, if you have any.
-# COPY requirements.txt  .
-# RUN  pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
+# You can install your application's other dependencies listed in `requirements.txt`.
+COPY requirements.txt .
+RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
 
 # Create a symlink for your application's entrypoint,
 # so we can use `/app.py` to refer it
@@ -69,6 +69,9 @@ ENTRYPOINT [ "/var/lang/bin/python3.8" ]
 CMD [ "/app.py" ]
 ```
 
+See [https://docs.aws.amazon.com/lambda/latest/dg/images-create.html#images-create-from-base](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html#images-create-from-base) 
+for more details on building your custom image.
+
 <!-- anchor_start:footer -->
 This crate is part of the [AWS SDK for Rust](https://awslabs.github.io/aws-sdk-rust/) and the [smithy-rs](https://github.com/awslabs/smithy-rs) code generator. In most cases, it should not be used directly.
 <!-- anchor_end:footer -->

rust-runtime/aws-smithy-http-server-python/examples/.gitignore

@@ -1,3 +1,3 @@
 pokemon-service-client/
 pokemon-service-server-sdk/
-libpokemon_service_server_sdk.so
+wheels/

rust-runtime/aws-smithy-http-server-python/examples/Makefile

@@ -1,56 +1,65 @@
-OS := $(shell uname -s)
 SRC_DIR := $(shell git rev-parse --show-toplevel)
 CUR_DIR := $(shell pwd)
 GRADLE := $(SRC_DIR)/gradlew
+WHEELS := $(CUR_DIR)/wheels
 SERVER_SDK_DST := $(CUR_DIR)/pokemon-service-server-sdk
 CLIENT_SDK_DST := $(CUR_DIR)/pokemon-service-client
 SERVER_SDK_SRC := $(SRC_DIR)/codegen-server-test/python/build/smithyprojections/codegen-server-test-python/pokemon-service-server-sdk/rust-server-codegen-python
 CLIENT_SDK_SRC := $(SRC_DIR)/codegen-client-test/build/smithyprojections/codegen-client-test/pokemon-service-client/rust-codegen
 
-SHARED_LIBRARY_DST := $(CUR_DIR)/libpokemon_service_server_sdk.so
-ifeq ($(OS), Darwin)
-	DEBUG_SHARED_LIBRARY_SRC := $(SRC_DIR)/target/debug/libpokemon_service_server_sdk.dylib
-	RELEASE_SHARED_LIBRARY_SRC := $(SRC_DIR)/target/release/libpokemon_service_server_sdk.dylib
-else
-	DEBUG_SHARED_LIBRARY_SRC := $(SRC_DIR)/target/debug/libpokemon_service_server_sdk.so
-	RELEASE_SHARED_LIBRARY_SRC := $(SRC_DIR)/target/release/libpokemon_service_server_sdk.so
-endif
+HAS_MATURIN := $(shell command -v maturin 2> /dev/null)
 
 all: codegen
 
 codegen:
 	$(GRADLE) --project-dir $(SRC_DIR) -P modules='pokemon-service-server-sdk,pokemon-service-client' :codegen-client-test:assemble :codegen-server-test:python:assemble
-	mkdir -p $(SERVER_SDK_DST) $(CLIENT_SDK_DST)
+	mkdir -p $(SERVER_SDK_DST) $(CLIENT_SDK_DST) $(WHEELS)
 	cp -av $(SERVER_SDK_SRC)/* $(SERVER_SDK_DST)/
 	cp -av $(CLIENT_SDK_SRC)/* $(CLIENT_SDK_DST)/
 
-clippy: codegen
-	cargo clippy
+ensure-maturin:
+ifndef HAS_MATURIN
+    $(error "maturin is not available; please install it via 'pip install maturin' or 'cargo install maturin'")
+endif
 
-build: codegen
-	cargo build
-	ln -sf $(DEBUG_SHARED_LIBRARY_SRC) $(SHARED_LIBRARY_DST)
+# Note on `--compatibility linux`: Maturin by default uses `manylinux_x_y` but it is not supported 
+# by our current CI version (3.7.10), we can drop `--compatibility linux` when we switch to higher Python version.
+# For more detail: https://github.com/pypa/manylinux
+build-wheel: ensure-maturin codegen
+	maturin build --manifest-path $(SERVER_SDK_DST)/Cargo.toml --out $(WHEELS) --compatibility linux
 
-py_check: build
-	mypy pokemon_service.py
+build-wheel-release: ensure-maturin codegen
+	maturin build --manifest-path $(SERVER_SDK_DST)/Cargo.toml --out $(WHEELS) --compatibility linux --release
+
+install-wheel:
+	find $(WHEELS) -type f -name '*.whl' | xargs python3 -m pip install --user --force-reinstall
+
+build: build-wheel install-wheel
 
-release: codegen
-	cargo build --release
-	ln -sf $(RELEASE_SHARED_LIBRARY_SRC) $(SHARED_LIBRARY_DST)
+release: build-wheel-release install-wheel
 
 run: build
 	python3 $(CUR_DIR)/pokemon_service.py
 
+run-release: release
+	python3 $(CUR_DIR)/pokemon_service.py
+
+py-check: build
+	mypy pokemon_service.py
+
 test: build
 	cargo test
 
+clippy: codegen
+	cargo clippy
+
 doc-open: codegen
 	cargo doc --no-deps --open
 
 clean:
 	cargo clean || echo "Unable to run cargo clean"
 
 distclean: clean
-	rm -rf $(SERVER_SDK_DST) $(CLIENT_SDK_DST) $(CUR_DIR)/Cargo.lock $(SHARED_LIBRARY_DST)
+	rm -rf $(SERVER_SDK_DST) $(CLIENT_SDK_DST) $(WHEELS) $(CUR_DIR)/Cargo.lock
 
 .PHONY: all

rust-runtime/aws-smithy-http-server-python/examples/pokemon_service.py

@@ -10,32 +10,32 @@
 from dataclasses import dataclass
 from typing import List, Optional, Callable, Awaitable
 
-from libpokemon_service_server_sdk import App
-from libpokemon_service_server_sdk.tls import TlsConfig  # type: ignore
-from libpokemon_service_server_sdk.aws_lambda import LambdaContext  # type: ignore
-from libpokemon_service_server_sdk.error import ResourceNotFoundException  # type: ignore
-from libpokemon_service_server_sdk.input import (  # type: ignore
+from pokemon_service_server_sdk import App
+from pokemon_service_server_sdk.tls import TlsConfig  # type: ignore
+from pokemon_service_server_sdk.aws_lambda import LambdaContext  # type: ignore
+from pokemon_service_server_sdk.error import ResourceNotFoundException  # type: ignore
+from pokemon_service_server_sdk.input import (  # type: ignore
     DoNothingInput,
     GetPokemonSpeciesInput,
     GetServerStatisticsInput,
     CheckHealthInput,
     StreamPokemonRadioInput,
 )
-from libpokemon_service_server_sdk.logging import TracingHandler  # type: ignore
-from libpokemon_service_server_sdk.middleware import (  # type: ignore
+from pokemon_service_server_sdk.logging import TracingHandler  # type: ignore
+from pokemon_service_server_sdk.middleware import (  # type: ignore
     MiddlewareException,
     Response,
     Request,
 )
-from libpokemon_service_server_sdk.model import FlavorText, Language  # type: ignore
-from libpokemon_service_server_sdk.output import (  # type: ignore
+from pokemon_service_server_sdk.model import FlavorText, Language  # type: ignore
+from pokemon_service_server_sdk.output import (  # type: ignore
     DoNothingOutput,
     GetPokemonSpeciesOutput,
     GetServerStatisticsOutput,
     CheckHealthOutput,
     StreamPokemonRadioOutput,
 )
-from libpokemon_service_server_sdk.types import ByteStream  # type: ignore
+from pokemon_service_server_sdk.types import ByteStream  # type: ignore
 
 # Logging can bee setup using standard Python tooling. We provide
 # fast logging handler, Tracingandler based on Rust tracing crate.

tools/Dockerfile

@@ -52,6 +52,8 @@ ARG cargo_udeps_version=0.1.35
 ARG cargo_hack_version=0.5.23
 ARG cargo_minimal_versions_version=0.1.8
 ARG cargo_check_external_types_version=0.1.6
+# Maturin is needed for Python SSDK
+ARG maturin_version=0.14.1
 ENV RUSTUP_HOME=/opt/rustup \
     CARGO_HOME=/opt/cargo \
     PATH=/opt/cargo/bin/:${PATH} \
@@ -100,6 +102,7 @@ RUN set -eux; \
     cargo install cargo-hack --locked --version ${cargo_hack_version}; \
     cargo install cargo-minimal-versions --locked --version ${cargo_minimal_versions_version}; \
     cargo install cargo-check-external-types --locked --version ${cargo_check_external_types_version}; \
+    cargo install maturin --locked --version ${maturin_version}; \
     if [[ "${checkout_smithy_rs_tools}" == "true" ]]; then \
         git clone https://github.com/awslabs/smithy-rs.git; \
         cd smithy-rs; \