Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
run-shader-validation: ${{ github.event_name != 'workflow_dispatch' || inputs.validate-shaders == true }}
run-shader-tests: ${{ github.event_name != 'workflow_dispatch' || inputs.validate-shaders == true }}
cache-key-suffix: ${{ inputs.cache-key-suffix || '' }}
run-vs2022: ${{ github.event_name == 'workflow_dispatch' && github.ref_type != 'tag' }}
run-vs2022: ${{ github.ref_type != 'tag' }}

feature-audit:
name: Feature Version Audit (Release)
Expand Down
197 changes: 166 additions & 31 deletions .github/workflows/upload-nexus.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -145,20 +145,17 @@ jobs:
fi
python tools/feature_version_audit.py "${ARGS[@]}"

# Inject GitHub release body as the core changelog.
# Fetch releases list as raw JSON and let Python search it — avoids
# jq version quirks where array[0] on an empty array produces no output
# instead of null, leaving the file empty and breaking json.load.
gh api "repos/$GITHUB_REPOSITORY/releases" \
2>/dev/null > releases.json || echo '[]' > releases.json
# Fetch the specific release by tag — avoids pagination issues with
# the releases list endpoint (default page size is 30).
gh api "repos/$GITHUB_REPOSITORY/releases/tags/$RELEASE_TAG" \
2>/dev/null > release.json || echo '{}' > release.json
python -c "
import json, os
with open('releases.json') as f:
releases = json.load(f)
if not isinstance(releases, list):
releases = []
tag = os.environ.get('RELEASE_TAG', '')
body = next((r.get('body') or '' for r in releases if r.get('tag_name') == tag), '')
with open('release.json') as f:
release = json.load(f)
if not isinstance(release, dict):
release = {}
body = release.get('body') or ''
with open('nexus-matrix-raw.json') as f:
data = json.load(f)
for row in data:
Expand All @@ -170,11 +167,26 @@ jobs:
"

python -c "
import json
import json, os, fnmatch
with open('nexus-matrix-raw.json') as f:
data = json.load(f)
# Load asset names from the release to skip uploads where the
# artifact was not included — applies to all rows including core.
try:
with open('release.json') as f:
release = json.load(f)
if not isinstance(release, dict):
release = {}
except Exception:
release = {}
asset_names = [a['name'] for a in release.get('assets', [])]
def artifact_in_release(row):
pat = row.get('artifact_pattern', '')
return bool(pat) and any(fnmatch.fnmatch(n, pat) for n in asset_names)
# Full matrix for artifact prep; upload matrix excludes auto_upload=false
upload_data = [row for row in data if row.get('auto_upload', True)]
# and features whose standalone artifact is absent from the release.
upload_data = [row for row in data
if row.get('auto_upload', True) and artifact_in_release(row)]
has_uploads = bool(upload_data)
with open('nexus-matrix.json', 'w') as f:
json.dump({'include': data}, f)
Expand Down Expand Up @@ -211,7 +223,7 @@ jobs:
- name: Download GitHub release assets
run: |
mkdir -p release-assets
gh release download "${{ needs.prepare-nexus-matrix.outputs.tag }}" --pattern "*.7z" --dir release-assets
gh release download "${{ needs.prepare-nexus-matrix.outputs.tag }}" --repo "$GITHUB_REPOSITORY" --pattern "*.7z" --dir release-assets
ls -lah release-assets
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down Expand Up @@ -262,6 +274,92 @@ jobs:
fi
fi

check-nexus-versions:
name: Dry Run — Check Nexus Versions
needs: prepare-nexus-matrix
if: >
needs.prepare-nexus-matrix.outputs.dry_run == 'true' &&
needs.prepare-nexus-matrix.outputs.has_uploads == 'true'
runs-on: ubuntu-latest
steps:
- name: Query Nexus for existing versions
run: |
python3 << 'PYEOF'
import json, os, urllib.request, urllib.error

rows = json.loads(os.environ['UPLOAD_MATRIX']).get('include', [])
rows = [r for r in rows if not r.get('skip')]
version = os.environ.get('VERSION', '')
game_id = os.environ.get('NEXUS_GAME_ID', 'skyrimspecialedition')
api_key = os.environ.get('UNEX_APIKEY', '')
summary = open(os.environ['GITHUB_STEP_SUMMARY'], 'a')

def w(line=''):
print(line)
summary.write(line + '\n')

w('## Dry Run — Nexus Version Check')
w()
if not api_key:
w('> **Warning:** UNEX_APIKEY is not set — cannot query Nexus.')
w()
w('| Feature | Planned | Status | Versions on Nexus |')
w('|---------|---------|--------|-------------------|')

api_ok = False
for row in rows:
name = row.get('name', '')
mod_id = str(row.get('nexus_mod_id', ''))
planned = row.get('mod_version') or version
label = row.get('mod_filename', name)
url = f'https://www.nexusmods.com/{game_id}/mods/{mod_id}'
link = f'[{label}]({url})'

if not api_key or not mod_id:
w(f'| {link} | `{planned}` | ⚠️ skipped | — |')
continue

api_url = f'https://api.nexusmods.com/v1/games/{game_id}/mods/{mod_id}/files.json'
req = urllib.request.Request(api_url, headers={
'apikey': api_key,
'User-Agent': 'community-shaders-ci/1.0',
'Accept': 'application/json',
})
try:
with urllib.request.urlopen(req, timeout=15) as resp:
data = json.load(resp)
api_ok = True
files = data.get('files', [])
main = [f for f in files if f.get('category_name') == 'MAIN']
check = main if main else files
seen = []
for f in check:
v = f.get('version', '')
if v and v not in seen:
seen.append(v)
versions_str = ', '.join(f'`{v}`' for v in seen[-5:]) or '—'
status = '✅ exists' if planned in seen else '🆕 new'
w(f'| {link} | `{planned}` | {status} | {versions_str} |')
except urllib.error.HTTPError as e:
err = {401: '❌ auth failed', 403: '❌ forbidden'}.get(e.code, f'❌ HTTP {e.code}')
w(f'| {link} | `{planned}` | {err} | — |')
except Exception:
w(f'| {link} | `{planned}` | ❌ error | — |')

w()
if api_key:
if api_ok:
w('> ✓ UNEX_APIKEY validated.')
else:
w('> ⚠️ UNEX_APIKEY may be invalid — all API calls failed.')
summary.close()
PYEOF
env:
UPLOAD_MATRIX: ${{ needs.prepare-nexus-matrix.outputs.upload_matrix }}
VERSION: ${{ needs.prepare-nexus-matrix.outputs.version }}
NEXUS_GAME_ID: ${{ inputs.nexus_game_id || 'skyrimspecialedition' }}
UNEX_APIKEY: ${{ secrets.UNEX_APIKEY }}

upload-to-nexus:
# Reusable workflow call: runs-on is intentionally omitted (runner is defined by the
# called workflow). IDE schema validators may flag this as an error — it is valid syntax.
Expand All @@ -282,15 +380,16 @@ jobs:
tag_name: ${{ needs.prepare-nexus-matrix.outputs.tag }}
mod_version: ${{ matrix.mod_version || needs.prepare-nexus-matrix.outputs.version }}
mod_filename: ${{ matrix.mod_filename }}
changelog: ${{ matrix.changelog || inputs.changelog || '' }}
changelog: ${{ inputs.changelog || matrix.changelog || '' }}
check_existing: true
secrets:
UNEX_NEXUSMODS_SESSION_COOKIE: ${{ secrets.UNEX_NEXUSMODS_SESSION_COOKIE }}
UNEX_APIKEY: ${{ secrets.UNEX_APIKEY }}

upload-summary:
name: Upload Summary
runs-on: ubuntu-latest
needs: [prepare-nexus-matrix, prepare-artifacts, upload-to-nexus]
needs: [prepare-nexus-matrix, prepare-artifacts, upload-to-nexus, check-nexus-versions]
if: always() && needs.prepare-nexus-matrix.result == 'success'
steps:
- name: Check upload status
Expand All @@ -314,17 +413,53 @@ jobs:
- name: Report upload result
if: needs.prepare-nexus-matrix.outputs.dry_run != 'true' && needs.prepare-nexus-matrix.outputs.has_uploads == 'true'
run: |
RESULT="${{ needs.upload-to-nexus.result }}"
if [ "$RESULT" = "success" ]; then
echo "✓ All Nexus uploads completed successfully" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "- [Community Shaders](https://www.nexusmods.com/skyrimspecialedition/mods/86492)" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Some uploads failed or were skipped (result: $RESULT)" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "**Recovery steps:**" >> $GITHUB_STEP_SUMMARY
echo "1. Check the upload-to-nexus job logs for details" >> $GITHUB_STEP_SUMMARY
echo "2. Verify credentials: UNEX_NEXUSMODS_SESSION_COOKIE, UNEX_APIKEY" >> $GITHUB_STEP_SUMMARY
echo "3. Download failed artifacts from this workflow run and upload manually" >> $GITHUB_STEP_SUMMARY
echo "4. Or re-run this workflow with the same tag to retry" >> $GITHUB_STEP_SUMMARY
fi
python3 << 'PYEOF'
import json, os

result = os.environ.get('UPLOAD_RESULT', '')
rows = json.loads(os.environ['UPLOAD_MATRIX']).get('include', [])
rows = [r for r in rows if not r.get('skip')]
version = os.environ.get('VERSION', '')
game_id = os.environ.get('NEXUS_GAME_ID', 'skyrimspecialedition')
summary = open(os.environ['GITHUB_STEP_SUMMARY'], 'a')

def w(line=''):
summary.write(line + '\n')

if result == 'success':
w('✅ All Nexus uploads completed successfully')
w()
w('| Feature | Version | Nexus |')
w('|---------|---------|-------|')
for row in rows:
mod_id = str(row.get('nexus_mod_id', ''))
planned = row.get('mod_version') or version
label = row.get('mod_filename', row.get('name', ''))
url = f'https://www.nexusmods.com/{game_id}/mods/{mod_id}'
w(f'| [{label}]({url}) | `{planned}` | [View]({url}) |')
else:
w(f'⚠️ Some uploads may have failed (result: `{result}`)')
w()
w('The following uploads were planned. Re-run this workflow to retry,')
w('or download the artifact and upload manually to Nexus.')
w()
w('| Feature | Version | Nexus | Artifact |')
w('|---------|---------|-------|----------|')
for row in rows:
name = row.get('name', '')
mod_id = str(row.get('nexus_mod_id', ''))
planned = row.get('mod_version') or version
label = row.get('mod_filename', name)
url = f'https://www.nexusmods.com/{game_id}/mods/{mod_id}'
artifact = f'nexus-upload-{name}'
w(f'| [{label}]({url}) | `{planned}` | [View]({url}) | `{artifact}` |')
w()
w('Re-run is safe: already-uploaded versions will be automatically skipped.')

summary.close()
PYEOF
env:
UPLOAD_RESULT: ${{ needs.upload-to-nexus.result }}
UPLOAD_MATRIX: ${{ needs.prepare-nexus-matrix.outputs.upload_matrix }}
VERSION: ${{ needs.prepare-nexus-matrix.outputs.version }}
NEXUS_GAME_ID: ${{ inputs.nexus_game_id || 'skyrimspecialedition' }}
33 changes: 22 additions & 11 deletions tools/feature_version_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,23 +336,28 @@ def get_feature_changelog(feature_dir, feature_info, base_ref):
if src_dir.exists() and src_dir.is_dir():
paths.append(str(src_dir).replace("\\", "/"))
try:
# Use ASCII unit-separator (0x1f) between subject and body so BREAKING
# CHANGE footers are detected even when the subject lacks the '!' marker.
# Null bytes are not used because Windows subprocess rejects them.
raw = subprocess.check_output(
["git", "log", f"{base_ref}..{HEAD_REF}", "--pretty=%s", "--"] + paths,
["git", "log", f"{base_ref}..{HEAD_REF}", "--pretty=format:%s\x1f%b\x1f", "--"] + paths,
cwd=str(PROJECT_ROOT),
stderr=subprocess.DEVNULL,
).decode("utf-8", errors="replace")
except Exception:
return ""
parts = raw.split("\x1f")
seen = set()
lines = []
for msg in raw.splitlines():
msg = msg.strip()
if not msg or msg in seen:
for subject, body in zip(parts[::2], parts[1::2]):
subject = subject.strip()
if not subject or subject in seen:
continue
seen.add(msg)
if (RE_COMMIT_FEAT.match(msg) or RE_COMMIT_FIX.match(msg) or
RE_COMMIT_PERF.match(msg) or RE_COMMIT_BREAKING.search(msg)):
lines.append(f"- {msg}")
seen.add(subject)
full = subject + "\n" + body
if (RE_COMMIT_FEAT.match(subject) or RE_COMMIT_FIX.match(subject) or
RE_COMMIT_PERF.match(subject) or RE_COMMIT_BREAKING.search(full)):
lines.append(f"- {subject}")
return "\n".join(lines)

def apply_version_bump(ini_path, proposed_ver_str):
Expand Down Expand Up @@ -934,9 +939,8 @@ def sanitize_name(name):
if not mod_id:
continue
name = info['name']
artifact_pattern = info.get('artifact_pattern') or f'{name}-*.7z'

# Read INI metadata (version, filename, etc.)
# Read INI metadata first — mod_filename is needed to derive artifact_pattern.
ini_path = get_feature_ini(name) # Pass name as string for fuzzy matching
ini_metadata = {}
mod_version = None
Expand All @@ -949,6 +953,13 @@ def sanitize_name(name):
# Use mod_filename from INI if available, else from feature metadata, else use name
mod_filename = ini_metadata.get('mod_filename') or info.get('mod_filename') or name

# artifact_pattern: explicit INI value takes precedence; fallback derives the
# pattern from the display name using the cmake convention of replacing spaces
# with dots — e.g. "Cloud Shadows" → "Cloud.Shadows-*.7z".
artifact_pattern = (ini_metadata.get('artifact_pattern')
or info.get('artifact_pattern')
or f"{mod_filename.replace(' ', '.')}-*.7z")

# Auto-upload is opt-in; missing metadata should not enable uploads.
auto_upload = ini_metadata.get('auto_upload', False)

Expand All @@ -963,7 +974,7 @@ def sanitize_name(name):
if mod_version:
row['mod_version'] = mod_version
if base_ref:
feature_dir = FEATURES_DIR / name
feature_dir = find_feature_dir(name) or FEATURES_DIR / name
changelog = get_feature_changelog(feature_dir, info, base_ref)
if changelog:
row['changelog'] = changelog
Expand Down
Loading