add/edit scripts and workflow to upload test data to obj storage (lin…

…ode#1025)

.github/workflows/integration_tests.yml

@@ -16,6 +16,64 @@ jobs:
         with:
           go-version: 'stable'
       - run: go version
-      - run: make testacc
+
+      - name: Update system packages
+        run: sudo apt-get update -y
+
+      - name: Install system deps
+        run: sudo apt-get install -y build-essential
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install Python deps
+        run: pip3 install requests wheel boto3
+
+      - name: Set release version env
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+      - name: Install go-junit-report
+        run: go install github.com/jstemmer/go-junit-report/v2@latest
+
+      - run: |
+          timestamp=$(date +'%Y%m%d%H%M')
+          report_filename="${timestamp}_terraform_test_report.json"
+          make testacc TESTARGS="-json" | tee "$report_filename"
         env:
           LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }}
+
+      - name: Convert JSON Report to XML
+        run: |
+          filename=$(ls | grep -E '^[0-9]{12}_terraform_test_report\.json')
+
+          if [ -f "$filename" ]; then
+            go_junit_report_dir=$(go env GOPATH)/bin
+            export PATH="$PATH:$go_junit_report_dir"
+            xml_filename=$(echo "$filename" | sed 's/\.json$/.xml/')
+            go-junit-report < "$filename" > "$xml_filename"
+            echo "Conversion from JSON to XML completed successfully."
+          else
+            echo "JSON test report file not found."
+            exit 1
+          fi
+        env:
+          GO111MODULE: on
+
+      - name: Add additional information to XML report
+        run: |
+          filename=$(ls | grep -E '^[0-9]{12}_terraform_test_report\.xml$') 
+          python scripts/add_to_xml_test_report.py \
+          --branch_name "${{ env.RELEASE_VERSION }}" \
+          --gha_run_id "$GITHUB_RUN_ID" \
+          --gha_run_number "$GITHUB_RUN_NUMBER" \
+          --xmlfile "${filename}"
+
+      - name: Upload test results to bucket
+        run: |
+          report_filename=$(ls | grep -E '^[0-9]{12}_terraform_test_report\.xml$')
+          python3 scripts/test_report_upload_script.py "${report_filename}"
+        env:
+          LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }}
+          LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }}

scripts/add_to_xml_test_report.py

@@ -0,0 +1,37 @@
+import argparse
+import xml.etree.ElementTree as ET
+
+# Parse command-line arguments
+parser = argparse.ArgumentParser(description='Modify XML with workflow information')
+parser.add_argument('--branch_name', required=True)
+parser.add_argument('--gha_run_id', required=True)
+parser.add_argument('--gha_run_number', required=True)
+parser.add_argument('--xmlfile', required=True)  # Added argument for XML file path
+
+args = parser.parse_args()
+
+# Open and parse the XML file
+xml_file_path = args.xmlfile
+tree = ET.parse(xml_file_path)
+root = tree.getroot()
+
+# Create new elements for the information
+branch_name_element = ET.Element('branch_name')
+branch_name_element.text = args.branch_name
+
+gha_run_id_element = ET.Element('gha_run_id')
+gha_run_id_element.text = args.gha_run_id
+
+gha_run_number_element = ET.Element('gha_run_number')
+gha_run_number_element.text = args.gha_run_number
+
+# Add the new elements to the root of the XML
+root.append(branch_name_element)
+root.append(gha_run_id_element)
+root.append(gha_run_number_element)
+
+# Save the modified XML
+modified_xml_file_path = xml_file_path  # Overwrite it
+tree.write(modified_xml_file_path)
+
+print(f'Modified XML saved to {modified_xml_file_path}')

scripts/test_report_upload_script.py

@@ -0,0 +1,41 @@
+import boto3
+import sys
+import os
+from botocore.exceptions import NoCredentialsError
+
+ACCESS_KEY = os.environ.get('LINODE_CLI_OBJ_ACCESS_KEY')
+SECRET_KEY = os.environ.get('LINODE_CLI_OBJ_SECRET_KEY')
+BUCKET_NAME = 'dx-test-results'
+
+linode_obj_config = {
+    "aws_access_key_id": ACCESS_KEY,
+    "aws_secret_access_key": SECRET_KEY,
+    "endpoint_url": "https://us-southeast-1.linodeobjects.com",
+    "region_name": "us-southeast-1",
+}
+
+
+def upload_to_linode_object_storage(file_name):
+    try:
+        s3 = boto3.client('s3', **linode_obj_config)
+
+        s3.upload_file(Filename=file_name, Bucket=BUCKET_NAME, Key=file_name)
+
+        print(f'Successfully uploaded {file_name} to Linode Object Storage.')
+
+    except NoCredentialsError:
+        print('Credentials not available. Ensure you have set your AWS credentials.')
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print('Usage: python upload_to_linode.py <file_name>')
+        sys.exit(1)
+
+    file_name = sys.argv[1]
+
+    if not file_name:
+        print('Error: The provided file name is empty or invalid.')
+        sys.exit(1)
+
+    upload_to_linode_object_storage(file_name)