NH-14860 Fix sdist and wheel creation

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# VS Code
+.vscode/

Dockerfile

@@ -1,4 +1,4 @@
-# build development environment to locally build appoptics_apm Python agent and publish RC versions
+# build development environment to locally build solarwinds_apm Python agent and publish RC versions
 FROM quay.io/pypa/manylinux2014_x86_64
 
 # install packages need to build Ruby and dependencies need to build agent locally

Makefile

@@ -1,4 +1,4 @@
-#    Copyright 2021 SolarWinds, LLC
+#    Copyright 2022 SolarWinds, LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,6 +19,16 @@ SHELL=bash
 # By default, 'make' does nothing and only prints available options
 nothing:
 	@echo -e "\nHi! How can I help you?"
+	@echo -e "  - 'make package-and-publish':"
+	@echo -e "          Build the agent package distribution and upload it to packagecloud."
+	@echo -e "  - 'make package':"
+	@echo -e "          Build the agent package distribution (sdist and bdist (wheels))."
+	@echo -e "  - 'make sdist':"
+	@echo -e "          Just build the agent package in sdist format."
+	@echo -e "  - 'make manylinux-wheels':"
+	@echo -e "          Just build manylinux 64-bit Python wheels (bdist)."
+	@echo -e "  - 'make aws-lambda':"
+	@echo -e "          Build the AWS Lambda layer (zip archive)."
 	@echo -e "  - 'make wrapper':"
 	@echo -e "          Locally generate SWIG wrapper for C/C++ headers."
 	@echo -e "  - 'STAGING_OBOE=1 make wrapper':"
@@ -107,11 +117,105 @@ download-bson-headers:
 # download all required header and library files
 download-all: download-headers download-liboboe
 
+#----------------------------------------------------------------------------------------------------------------------#
+# check if SWIG is installed
+#----------------------------------------------------------------------------------------------------------------------#
+
+check-swig:
+	@echo -e "Is SWIG installed?"
+	@command -v swig >/dev/null 2>&1 || \
+		{ echo >&2 "Swig is required to build the distribution. Aborting."; exit 1;}
+	@echo -e "Yes."
+
+#----------------------------------------------------------------------------------------------------------------------#
+# recipes for building the package distribution
+#----------------------------------------------------------------------------------------------------------------------#
+
+wheel_tag := 'manylinux2014_x86_64'
+
+# Build the Python wrapper from liboboe headers inside build container
+wrapper: check-swig download-all
+	@echo -e "Generating SWIG wrapper for C/C++ headers."
+	@cd solarwinds_apm/extension && ./gen_bindings.sh
+
+# Create package source distribution archive
+sdist: wrapper
+	@echo -e "Generating python agent sdist package"
+	@python3.8 setup.py sdist
+	@echo -e "\nDone."
+
+# Build the Python agent package bdist (wheels) for 64 bit many linux systems (except Alpine).
+# The recipe builds the wheels for all Python versions available in the docker image, similarly to the example provided
+# in the corresponding repo of the Docker images: https://github.com/pypa/manylinux#example.
+manylinux-wheels: wrapper
+	@echo -e "Generating python agent package any-linux wheels for 64 bit systems"
+	@set -e; for PYBIN in /opt/python/*/bin; do "$${PYBIN}/pip" wheel . -w ./tmp_dist/ --no-deps; done
+	@echo -e "Tagging wheels with $(wheel_tag)"
+	@set -e; for whl in ./tmp_dist/*.whl; do auditwheel repair --plat $(wheel_tag) "$$whl" -w ./dist/; done
+	@rm -rf ./tmp_dist
+	@echo -e "\nDone."
+
+# Build the full Python agent distribution (sdist and wheels)
+package: sdist manylinux-wheels
+
+# Build the AWS lambda layer.
+# temporary target directory for AWS Lambda build artifacts
+# TODO cp39 and cp310
+target_dir := "./tmp/python"
+aws-lambda: wrapper
+	@if [ -f ./dist/solarwinds_apm_lambda.zip ]; then \
+		echo -e "Deleting old solarwinds_apm_lambda.zip"; \
+		rm ./dist/solarwinds_apm_lambda.zip; \
+	 fi
+	rm -rf ./tmp
+	@echo -e "Creating target directory ${target_dir} for AWS Lambda layer artifacts."
+	mkdir -p ${target_dir}
+	@echo -e "Install solarwinds_apm to be packed up in zip archive to target directory."
+	@/opt/python/cp38-cp38/bin/pip3.8 install . -t ${target_dir}
+	@echo -e "Removing non-lambda C-extension library files generated by pip install under target directory."
+	@rm ${target_dir}/solarwinds_apm/extension/*.so*
+	@echo -e "Building AWS Lambda version of C-extensions for all supported Python versions in target directory."
+	@set -e; for PYBIN in cp36-cp36m cp37-cp37m cp38-cp38; do /opt/python/$${PYBIN}/bin/python setup.py build_ext_lambda -b ${target_dir}; done
+	@echo -e "Copying AWS Lambda specific Oboe library liboboe-1.0-lambda-x86_64.so.0.0.0 into target directory."
+	@cp solarwinds_apm/extension/liboboe-1.0-lambda-x86_64.so.0.0.0 ${target_dir}/solarwinds_apm/extension/liboboe-1.0.so.0
+	@rm -rf ${target_dir}/*-info
+	@find ${target_dir} -type d -name '__pycache__' | xargs rm -rf
+	@if [[ ! -d dist ]]; then mkdir dist; fi
+	@pushd ./tmp && zip -r ../dist/solarwinds_apm_lambda.zip ./python && popd
+	@rm -rf ./tmp ./build
+	@echo -e "\nDone."
 
 #----------------------------------------------------------------------------------------------------------------------#
-# DEPRECATED: variable definitions and recipes for copying required header and library files from neighbouring local liboboe directory
+# variable and recipe definitions for distribution/ publishing workflow
 #----------------------------------------------------------------------------------------------------------------------#
 
+# current version of solarwinds_apm package
+package_version :=$(shell grep __version__ ./solarwinds_apm/version.py | cut -d= -f 2 | tr -d ' "')
+# Build package from working tree and upload to packagecloud
+package-and-publish: package upload-to-packagecloud clean
+	@echo -e "Built the agent package from local code and uploaded to packagecloud.io"
+
+# Go through the build process and publish AWS Lambda layer RC version
+publish-lambda-layer-rc: aws-lambda
+	@python3.8 publish_lambda_layer.py rc
+	@echo -e "Done: Built the AWS Lambda layer and uploaded it to AWS."
+
+# Upload built package to packagecloud
+pc_repo := "solarwinds/solarwinds-apm-python/python"
+pc_token := $(PACKAGECLOUD_TOKEN)
+upload-to-packagecloud:
+	@if [ -z $(pc_token) ]; then \
+		echo "PACKAGECLOUD_TOKEN environment variable is not set but required to upload to package cloud."; \
+		exit 1; \
+	 fi
+	@PACKAGECLOUD_TOKEN=$(pc_token) /usr/local/rvm/bin/rvm 2.5.1 do package_cloud push $(pc_repo) dist/solarwinds_apm-$(package_version).tar.gz
+	@PACKAGECLOUD_TOKEN=$(pc_token) /usr/local/rvm/bin/rvm 2.5.1 do package_cloud push $(pc_repo) dist/solarwinds_apm-$(package_version)*.whl
+
+#----------------------------------------------------------------------------------------------------------------------#
+# recipes for local development
+#----------------------------------------------------------------------------------------------------------------------#
+
+# neighbouring local liboboe directory
 OTELOBOEREPO := /code/solarwinds-apm-liboboe/liboboe
 
 # Copy the pre-compiled liboboe shared library from source specified in OTELOBOEREPO
@@ -154,8 +258,13 @@ copy-bson-headers:
 # copy artifacts from local oboe
 copy-all: copy-headers copy-liboboe
 
+# Build the Python wrapper from liboboe headers inside build container
+wrapper-from-local: check-swig copy-all
+	@echo -e "Generating SWIG wrapper for C/C++ headers, from local neighbouring oboe checkout"
+	@cd solarwinds_apm/extension && ./gen_bindings.sh
+
 #----------------------------------------------------------------------------------------------------------------------#
-# variable definitions and recipes for testing and linting
+# variable definitions and recipes for testing, linting, cleanup
 #----------------------------------------------------------------------------------------------------------------------#
 
 # Example: make tox OPTIONS="-e py37-nh-staging"
@@ -168,32 +277,6 @@ format:
 lint:
 	@echo -e "Not implemented."
 
-#----------------------------------------------------------------------------------------------------------------------#
-# recipes for building the package distribution
-#----------------------------------------------------------------------------------------------------------------------#
-# Check if SWIG is installed
-check-swig:
-	@echo -e "Is SWIG installed?"
-	@command -v swig >/dev/null 2>&1 || \
-		{ echo >&2 "Swig is required to build the distribution. Aborting."; exit 1;}
-	@echo -e "Yes."
-
-# Build the Python wrapper from liboboe headers inside build container
-wrapper: check-swig download-all
-	@echo -e "Generating SWIG wrapper for C/C++ headers."
-	@cd solarwinds_apm/extension && ./gen_bindings.sh
-
-# Build the Python wrapper from liboboe headers inside build container
-wrapper-from-local: check-swig copy-all
-	@echo -e "Generating SWIG wrapper for C/C++ headers, from local neighbouring oboe checkout"
-	@cd solarwinds_apm/extension && ./gen_bindings.sh
-
-# Create package source distribution archive
-sdist: wrapper
-	@echo -e "Generating python agent sdist package"
-	@python3.8 setup.py sdist
-	@echo -e "\nDone."
-
 # clean up everything.
 clean:
 	@echo -e "Cleaning intermediate files."
@@ -210,4 +293,4 @@ clean:
 	@rm -f .coverage.*
 	@echo -e "Done."
 
-.PHONY: nothing verify-oboe-version download-liboboe download-headers download-bson-headers download-all copy-liboboe copy-headers copy-bson-headers copy-all test test-all format lint check-swig wrapper wrapper-from-local sdist clean
+.PHONY: nothing verify-oboe-version download-liboboe download-headers download-bson-headers download-all check-swig wrapper sdist manylinux-wheels package aws-lambda package-and-publish publish-lambda-layer-rc upload-to-packagecloud copy-liboboe copy-headers copy-bson-headers copy-all wrapper-from-local tox format lint clean