Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test mentorship pr #5622

Closed
wants to merge 42 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
d1d92c3
test
mirnawong1 Apr 8, 2024
e98be1e
add python script and gha yaml for pr summary
mirnawong1 May 21, 2024
9949b27
update to edited
mirnawong1 May 21, 2024
3a7250e
Merge branch 'current' into mentorship
mirnawong1 May 21, 2024
b258cc7
move to root
mirnawong1 May 21, 2024
2ea3bdd
Merge branch 'current' into mentorship
matthewshaver May 29, 2024
900891a
add gha check
mirnawong1 Jun 3, 2024
7aac4ac
Merge branch 'current' into mentorship
mirnawong1 Jun 3, 2024
6bf4ead
update
mirnawong1 Jun 3, 2024
f578a0c
Merge branch 'mentorship' of https://github.com/dbt-labs/docs.getdbt.…
mirnawong1 Jun 3, 2024
f040cee
update
mirnawong1 Jun 3, 2024
0dbb29f
remove
mirnawong1 Jun 3, 2024
1a435ec
remove github yaml
mirnawong1 Jun 3, 2024
0f82cc4
add diff jsons
mirnawong1 Jun 3, 2024
d31bf54
update
mirnawong1 Jun 3, 2024
e4959f6
update to all yaml
mirnawong1 Jun 3, 2024
f985b05
update
mirnawong1 Jun 3, 2024
16128c5
update
mirnawong1 Jun 3, 2024
57d0679
update
mirnawong1 Jun 3, 2024
1dd1673
update
mirnawong1 Jun 3, 2024
1b35865
update
mirnawong1 Jun 3, 2024
993abb2
update
mirnawong1 Jun 3, 2024
95d95a5
add
mirnawong1 Jun 3, 2024
bd0d2b9
debug
mirnawong1 Jun 3, 2024
3cdc812
add
mirnawong1 Jun 3, 2024
53bf591
update
mirnawong1 Jun 3, 2024
b97389f
update
mirnawong1 Jun 3, 2024
f9a3b0d
update
mirnawong1 Jun 3, 2024
ca9a48e
update
mirnawong1 Jun 3, 2024
f10c5ce
update
mirnawong1 Jun 3, 2024
859d770
update
mirnawong1 Jun 3, 2024
2eab5be
update
mirnawong1 Jun 3, 2024
402ebf2
update
mirnawong1 Jun 3, 2024
d114171
add
mirnawong1 Jun 3, 2024
cfe9f36
update
mirnawong1 Jun 3, 2024
06ffb9b
add
mirnawong1 Jun 3, 2024
7fe64ad
update
mirnawong1 Jun 3, 2024
ee5a09c
update
mirnawong1 Jun 3, 2024
7c83127
update
mirnawong1 Jun 3, 2024
7b217c9
updates
mirnawong1 Jun 3, 2024
24cd16e
update
mirnawong1 Jun 3, 2024
2777653
Merge branch 'current' into mentorship
mirnawong1 Jun 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .github/workflows/validate_snippets.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: Validate code snippets

on:
pull_request:
types: [opened, edited, synchronize]

jobs:
validate_snippets:
runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v2

- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.x'

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install requests jsonschema pyyaml

- name: Run validation script
env:
SNIPPET_VALIDATION: ${{ secrets.SNIPPET_VALIDATION }}
run: |
python website/validate_snippets.py

- name: Display validation results
if: failure()
run: cat website/validation_results.txt
2 changes: 1 addition & 1 deletion website/src/components/quickstartGuideList/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -129,4 +129,4 @@ function QuickstartList({ quickstartData }) {
)
}

export default QuickstartList;
export default QuickstartList;
100 changes: 100 additions & 0 deletions website/validate_snippets.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import os
import json
import requests
import yaml
from jsonschema import validate, ValidationError
import re

SCHEMA_URLS = {
"dbt_project": "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/dbt_project-latest.json",
"dbt_cloud": "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/dbt_cloud-latest.json",
"dbt_yml": "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/dbt_yml_files-latest.json",
"dependencies": "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/dependencies-latest.json",
"packages": "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/packages-latest.json",
"selectors": "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/selectors-latest.json",
}

def fetch_schemas():
schemas = {}
for name, url in SCHEMA_URLS.items():
response = requests.get(url)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
print(f"Failed to fetch schema {name}: {e}")
print(f"Response: {response.text}")
raise
schemas[name] = response.json()
return schemas

def fetch_code_snippets(pr_number, repo_owner, repo_name):
url = f'https://api.github.com/repos/{repo_owner}/{repo_name}/pulls/{pr_number}/files'
headers = {'Authorization': f'token {os.getenv("SNIPPET_VALIDATION")}'}
response = requests.get(url, headers=headers)
response.raise_for_status()
files = response.json()

snippets = []
for file in files:
if file['filename'].endswith('.md'):
patch = file['patch']
yaml_snippets = re.findall(r'```(yaml|yml)\n(.*?)```', patch, re.DOTALL)
for snippet in yaml_snippets:
cleaned_snippet = "\n".join(line[1:] if line.startswith(('+', '-')) else line for line in snippet[1].split('\n'))
snippets.append((file['filename'], cleaned_snippet))
return snippets

def identify_schema(snippet):
if "dbt_project" in snippet:
return "dbt_project"
elif "dbt_cloud" in snippet:
return "dbt_cloud"
elif "dependencies" in snippet:
return "dependencies"
elif "packages" in snippet:
return "packages"
elif "selectors" in snippet:
return "selectors"
return "dbt_yml"

def validate_snippets(snippets, schemas):
results = []
for filename, snippet in snippets:
try:
data = yaml.safe_load(snippet)
schema_name = identify_schema(snippet)
schema = schemas.get(schema_name)
if schema:
validate(instance=data, schema=schema)
results.append((filename, snippet, "Valid"))
else:
results.append((filename, snippet, "Unknown schema type"))
except ValidationError as e:
results.append((filename, snippet, f"Invalid: {e.message}"))
except yaml.YAMLError as e:
results.append((filename, snippet, f"Invalid YAML: {str(e)}"))
return results

def main():
repo_owner = "dbt-labs"
repo_name = "docs.getdbt.com"

with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as f:
event = json.load(f)

pr_number = event['pull_request']['number']

schemas = fetch_schemas()
snippets = fetch_code_snippets(pr_number, repo_owner, repo_name)

results = validate_snippets(snippets, schemas)

with open('website/validation_results.txt', 'w') as f:
for filename, snippet, result in results:
f.write(f"File: {filename}\nSnippet:\n{snippet}\nResult: {result}\n\n")

if any("Invalid" in result for _, _, result in results):
exit(1)

if __name__ == "__main__":
main()
Loading