diff --git a/.github/workflows/deploy_to_prod.yml b/.github/workflows/deploy_to_prod.yml new file mode 100644 index 0000000..8853f77 --- /dev/null +++ b/.github/workflows/deploy_to_prod.yml @@ -0,0 +1,54 @@ +# This workflow will update the code for the production environment + +on: + push: + branches: ["release"] + +name: Deploy to production + +jobs: + build_and_deploy: + name: Deploy + environment: + name: Production + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + + - name: Set up SAM + uses: aws-actions/setup-sam@v1 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ap-south-1 + + - name: Build + run: sam build --use-container -t templates/prod.yaml + + - name: Deploy + env: + DYNAMODB_URL: ${{ secrets.DYNAMODB_URL }} + DYNAMODB_REGION: ${{ secrets.DYNAMODB_REGION }} + DYNAMODB_ACCESS_KEY: ${{ secrets.DYNAMODB_ACCESS_KEY }} + DYNAMODB_SECRET_KEY: ${{ secrets.DYNAMODB_SECRET_KEY }} +run: > + sam deploy + --stack-name ReportingProduction + --s3-bucket reporting-engine-production + --no-confirm-changeset + --no-fail-on-empty-changeset + --region ap-south-1 + --capabilities CAPABILITY_IAM + --parameter-overrides + DynamodbUrl=$DYNAMODB_URL + DynamodbRegion=$DYNAMODB_REGION + DynamodbAccessKey=$DYNAMODB_ACCESS_KEY + DynamodbSecretKey=$DYNAMODB_SECRET_KEY \ No newline at end of file diff --git a/.github/workflows/deploy_to_staging.yml b/.github/workflows/deploy_to_staging.yml new file mode 100644 index 0000000..b4d2ffe --- /dev/null +++ b/.github/workflows/deploy_to_staging.yml @@ -0,0 +1,55 @@ +# This workflow will update the code for the staging environment + +on: + pull_request: + push: + branches: ["main"] + +name: Deploy to staging + +jobs: + build_and_deploy: + name: Deploy + environment: + name: Staging + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + + - name: Set up SAM + uses: aws-actions/setup-sam@v1 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ap-south-1 + + - name: Build + run: sam build --use-container -t templates/staging.yaml + + - name: Deploy + env: + DYNAMODB_URL: ${{ secrets.DYNAMODB_URL }} + DYNAMODB_REGION: ${{ secrets.DYNAMODB_REGION }} + DYNAMODB_ACCESS_KEY: ${{ secrets.DYNAMODB_ACCESS_KEY }} + DYNAMODB_SECRET_KEY: ${{ secrets.DYNAMODB_SECRET_KEY }} + run: > + sam deploy + --stack-name ReportingStaging + --s3-bucket reporting-engine-staging + --no-confirm-changeset + --no-fail-on-empty-changeset + --region ap-south-1 + --capabilities CAPABILITY_IAM + --parameter-overrides + DynamodbUrl=$DYNAMODB_URL + DynamodbRegion=$DYNAMODB_REGION + DynamodbAccessKey=$DYNAMODB_ACCESS_KEY + DynamodbSecretKey=$DYNAMODB_SECRET_KEY \ No newline at end of file diff --git a/README.md b/README.md index fd27443..ef9cb18 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Welcome to the AF Reporting Engine! -## Installation and First Run +## Setting up the Reporting Engine locally 0. Install pre-commit hooks in the repo ``` @@ -30,10 +30,30 @@ python generate_table This will create the the `student_quiz_reports` table. -## Accessing things +### Accessing things DynamoDB Admin: localhost:8001 Reporting FastAPI Server: localhost:5050 (docs and API tryout at localhost:5050/docs) DynamoDB server: localhost:8000 (we won't access this directly) + + +## Connect with DynamoDB Server +0. Install pre-commit hooks in the repo +``` +pip install pre-commit +pre-commit install +``` +1. Obtain credentials to replace local keys in `.env.local` file from repository owners. + +2. Run the following to get app at `localhost:5050/docs` +``` +cd app; uvicorn main:app --port 5050 --reload +``` +### Deployment +We deploy our FastAPI instance on AWS Lambda which is triggered via an API Gateway. In order to automate the process, we use AWS SAM, which creates the stack required for deployment and updates it as needed with just a couple of commands and without having to do anything manually on the AWS GUI. Refer to this [blog](https://www.eliasbrange.dev/posts/deploy-fastapi-on-aws-part-1-lambda-api-gateway/) post for more details. + +The actual deployment happens through Github Actions. Look at `.github/workflows/deploy_to_staging.yml` to understand the deployment to Staging and `.github/workflows/deploy_to_prod.yml` for Production. + +The details of the AWS Lambda instances are described in `templates/prod.yaml` and `templates/staging.yaml`. \ No newline at end of file diff --git a/app/internal/db.py b/app/internal/db.py index e4a0077..1073aa9 100644 --- a/app/internal/db.py +++ b/app/internal/db.py @@ -2,7 +2,13 @@ from dotenv import load_dotenv import os -load_dotenv(".env.local") +# when running app locally -- use load_dotenv +# when running app via gh actions -- variables already exist via secrets +# so no need to load_dotenv +if not all(key in os.environ for key in + ["DYNAMODB_URL", "DYNAMODB_REGION", + "DYNAMODB_ACCESS_KEY", "DYNAMODB_SECRET_KEY"]): + load_dotenv("../.env.local") def initialize_db(): diff --git a/app/main.py b/app/main.py index 469a6fb..4b1cccc 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,5 @@ -import uvicorn from fastapi import FastAPI +from mangum import Mangum from internal.db import initialize_db @@ -12,7 +12,7 @@ app = FastAPI() -app.mount("/static", StaticFiles(directory="app/static"), name="static") +app.mount("/static", StaticFiles(directory="static"), name="static") db = initialize_db() @@ -29,8 +29,7 @@ @app.get("/") def index(): - return "Hello World!" + return "Hello World! Welcome to Reporting Engine!" -if __name__ == "__main__": - uvicorn.run("main:app", host="0.0.0.0", port=5050, log_level="info", reload=True) +handler = Mangum(app) diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 0000000..10810a7 --- /dev/null +++ b/app/requirements.txt @@ -0,0 +1,24 @@ +anyio==3.6.1 +autopep8==1.6.0 +boto3==1.24.35 +botocore==1.27.35 +click==8.1.3 +fastapi==0.79.0 +h11==0.13.0 +idna==3.3 +Jinja2==3.1.2 +jmespath==1.0.1 +MarkupSafe==2.1.1 +pycodestyle==2.8.0 +pydantic==1.9.1 +python-dateutil==2.8.2 +python-dotenv==0.20.0 +s3transfer==0.6.0 +six==1.16.0 +sniffio==1.2.0 +starlette==0.19.1 +toml==0.10.2 +typing_extensions==4.3.0 +urllib3==1.26.10 +uvicorn==0.18.2 +mangum==0.14.1 diff --git a/app/routers/reports.py b/app/routers/reports.py index 025bdc5..ec768bb 100644 --- a/app/routers/reports.py +++ b/app/routers/reports.py @@ -23,7 +23,7 @@ def __init__( self, student_quiz_reports_controller: StudentQuizReportController ) -> None: self.__student_quiz_reports_controller = student_quiz_reports_controller - self._templates = Jinja2Templates(directory="app/templates") + self._templates = Jinja2Templates(directory="templates") @property def router(self): diff --git a/internal/__init__.py b/internal/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/internal/__main__.py b/internal/__main__.py deleted file mode 100644 index e1427c2..0000000 --- a/internal/__main__.py +++ /dev/null @@ -1,36 +0,0 @@ -from student_quiz_reports import ( - generate_student_quiz_reports, - drop_student_quiz_reports, -) -import boto3 -from dotenv import load_dotenv - -import os - -load_dotenv(".env.local") - - -def initialize_db(): - ddb = boto3.resource( - "dynamodb", - endpoint_url=os.environ.get("DYNAMODB_URL"), - region_name=os.environ.get("DYNAMODB_REGION"), - aws_access_key_id=os.environ.get("DYNAMODB_ACCESS_KEY"), - aws_secret_access_key=os.environ.get("DYNAMODB_SECRET_KEY"), - ) - - return ddb - - -def generate_tables(): - ddb = initialize_db() - generate_student_quiz_reports(ddb) - - -def drop_tables(): - ddb = initialize_db() - drop_student_quiz_reports(ddb) - - -if __name__ == "__main__": - generate_tables() diff --git a/internal/student_quiz_reports.py b/internal/student_quiz_reports.py deleted file mode 100644 index a803abd..0000000 --- a/internal/student_quiz_reports.py +++ /dev/null @@ -1,30 +0,0 @@ -def generate_student_quiz_reports(ddb): - ddb.create_table( - TableName="student_quiz_reports", - AttributeDefinitions=[ - {"AttributeName": "id", "AttributeType": "S"}, - {"AttributeName": "quiz_id", "AttributeType": "S"}, - {"AttributeName": "student_id", "AttributeType": "S"}, - ], - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - GlobalSecondaryIndexes=[ - { - "IndexName": "quiz_id", - "KeySchema": [ - {"AttributeName": "quiz_id", "KeyType": "HASH"}, - {"AttributeName": "student_id", "KeyType": "RANGE"}, - ], - "Projection": { - "ProjectionType": "ALL", - }, - }, - ], - BillingMode="PAY_PER_REQUEST", - ProvisionedThroughput={"ReadCapacityUnits": 10, "WriteCapacityUnits": 10}, - ) - print("Successfully created Student Quiz Reports Table") - - -def drop_student_quiz_reports(ddb): - table = ddb.Table("student_quiz_reports") - table.delete() diff --git a/requirements.txt b/requirements.txt index 8c197de..10810a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,4 @@ toml==0.10.2 typing_extensions==4.3.0 urllib3==1.26.10 uvicorn==0.18.2 +mangum==0.14.1 diff --git a/templates/prod.yaml b/templates/prod.yaml new file mode 100644 index 0000000..0bd2ceb --- /dev/null +++ b/templates/prod.yaml @@ -0,0 +1,46 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: Reporting Engine - Production - FastAPI on Lambda + +Parameters: + DynamodbUrl: + Type: String + Description: Url of Dynamodb instance + DynamodbRegion: + Type: String + Description: Region of Dynamodb instance + DynamodbAccessKey: + Type: String + Description: Access key credentials of Dynamodb instance + DynamodbSecretKey: + Type: String + Description: Secret key credentials of Dynamodb instance + +Resources: + Function: + Type: AWS::Serverless::Function + Properties: + FunctionName: "ReportingProduction" + CodeUri: ../app + Handler: main.handler + Runtime: python3.9 + Environment: + Variables: + DYNAMODB_URL: !Ref DynamodbUrl + DYNAMODB_REGION: !Ref DynamodbRegion + DYNAMODB_ACCESS_KEY: !Ref DynamodbAccessKey + DYNAMODB_SECRET_KEY: !Ref DynamodbSecretKey + Events: + Api: + Type: HttpApi + Properties: + ApiId: !Ref Api + + Api: + Type: AWS::Serverless::HttpApi + +Outputs: + ApiUrl: + Description: URL of your API + Value: + Fn::Sub: "https://${Api}.execute-api.${AWS::Region}.${AWS::URLSuffix}/" \ No newline at end of file diff --git a/templates/staging.yaml b/templates/staging.yaml new file mode 100644 index 0000000..fa6b919 --- /dev/null +++ b/templates/staging.yaml @@ -0,0 +1,46 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: Reporting Engine - Staging - FastAPI on Lambda + +Parameters: + DynamodbUrl: + Type: String + Description: Url of Dynamodb instance + DynamodbRegion: + Type: String + Description: Region of Dynamodb instance + DynamodbAccessKey: + Type: String + Description: Access key credentials of Dynamodb instance + DynamodbSecretKey: + Type: String + Description: Secret key credentials of Dynamodb instance + +Resources: + Function: + Type: AWS::Serverless::Function + Properties: + FunctionName: "ReportingStaging" + CodeUri: ../app + Handler: main.handler + Runtime: python3.9 + Environment: + Variables: + DYNAMODB_URL: !Ref DynamodbUrl + DYNAMODB_REGION: !Ref DynamodbRegion + DYNAMODB_ACCESS_KEY: !Ref DynamodbAccessKey + DYNAMODB_SECRET_KEY: !Ref DynamodbSecretKey + Events: + Api: + Type: HttpApi + Properties: + ApiId: !Ref Api + + Api: + Type: AWS::Serverless::HttpApi + +Outputs: + ApiUrl: + Description: URL of your API + Value: + Fn::Sub: "https://${Api}.execute-api.${AWS::Region}.${AWS::URLSuffix}/" \ No newline at end of file