first try

.github/workflows/CICD.yml

@@ -23,14 +23,16 @@ jobs:
         run: make test
       - name: format
         run: make format
-      # - name: Commit files
-      #   run: |
-      #     git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-      #     git config --local user.name "github-actions[bot]"
-      #     git add .
-      #     git commit -a -m "Add changes"
-      # - name: Push changes
-      #   uses: ad-m/github-push-action@master
-      #   with:
-      #     github_token: ${{ secrets.GITHUB_TOKEN }}
-      #     branch: ${{ github.ref }}
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      # Build and push Docker image
+      - name: Build and push Docker image
+        run: |
+          docker build -t ${{ secrets.DOCKER_USERNAME }}/databricks-file-pipeline .
+          docker tag ${{ secrets.DOCKER_USERNAME }}/databricks-file-pipeline:latest ${{ secrets.DOCKER_USERNAME }}/databricks-file-pipeline:latest
+          docker push ${{ secrets.DOCKER_USERNAME }}/databricks-file-pipeline:latest

Dockerfile

@@ -0,0 +1,17 @@
+# Base image
+FROM python:3.9-slim
+
+# Set working directory
+WORKDIR /app
+
+# Copy application files
+COPY app.py requirements.txt .env /app/
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose the port Flask will run on
+EXPOSE 5000
+
+# Command to run the application
+CMD ["python", "app.py"]

Makefile

@@ -1,14 +1,44 @@
-install: 
-	pip install --upgrade pip && pip install -r requirements.txt 
+# Application-specific variables
+APP_NAME = databricks-file-pipeline
+DOCKER_USER = 31good
+DOCKER_TAG = $(DOCKER_USER)/$(APP_NAME):latest
 
-format: 
-	black *.py
+# Default make targets
+all: install format lint test
+
+# Install dependencies
+install:
+	pip install --upgrade pip && pip install -r requirements.txt
 
+# Format Python files
+format:
+	black *.py mylib/*.py
+
+# Lint codebase
 lint:
-	#pylint --disable=R,C --ignore-patterns=test_.*?py $(wildcard *.py)
 	ruff check *.py mylib/*.py
 
-test: 
-	python -m pytest -cov=main test_main.py
+# Run tests with coverage
+test:
+	python -m pytest --cov=main test_main.py
 
-all: install format lint test
+# Build Docker image
+build:
+	docker build -t $(APP_NAME) .
+
+# Run Docker container
+run:
+	docker run -p 5000:5000 --env-file .env $(APP_NAME)
+
+# Tag Docker image
+tag:
+	docker tag $(APP_NAME) $(DOCKER_TAG)
+
+# Push Docker image to Docker Hub
+push: tag
+	docker push $(DOCKER_TAG)
+
+# Clean up Docker resources
+clean:
+	docker rmi -f $(APP_NAME) $(DOCKER_TAG)
+	docker rm -f $(APP_NAME)-container || true

app.py

@@ -0,0 +1,46 @@
+import os
+from flask import Flask, jsonify, request, send_file
+from dotenv import load_dotenv
+
+app = Flask(__name__)
+
+# Serve the homepage
+@app.route('/')
+def home():
+    return "Databricks File Checker and Image Display API"
+
+# Endpoint to check file status
+@app.route('/check-file', methods=['POST'])
+def check_file():
+    load_dotenv()
+    server_h = os.getenv("SERVER_HOSTNAME")
+    access_token = os.getenv("ACCESS_TOKEN")
+    url = f"https://{server_h}/api/2.0"
+
+    data = request.json
+    file_path = data.get("file_path")
+
+    if not file_path:
+        return jsonify({"error": "file_path is required"}), 400
+
+    headers = {'Authorization': f'Bearer {access_token}'}
+    try:
+        response = request.get(url + f"/dbfs/get-status?path={file_path}", headers=headers)
+        response.raise_for_status()
+        return jsonify({"file_exists": True, "file_path": file_path})
+    except Exception as e:
+        return jsonify({"file_exists": False, "error": str(e)})
+
+# Endpoint to serve the image
+@app.route('/display-image', methods=['GET'])
+def display_image():
+    image_path = "mylib/top_10_countries_alcohol_consumption.png"  # Adjust as per your Docker file structure
+    full_path = os.path.join("/Workspace/Shared/Allen_Wang_miniproj_11", image_path)
+
+    if os.path.exists(full_path):
+        return send_file(full_path, mimetype='image/png')
+    else:
+        return jsonify({"error": "Image not found"}), 404
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=5000)