Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions .github/workflows/autoqa-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,13 @@ jobs:
# Find the exe file in the artifact
$exeFile = Get-ChildItem -Path "${{ runner.temp }}/windows-artifact" -Recurse -Filter "*.exe" | Select-Object -First 1
if ($exeFile) {
Write-Host " Found local installer: $($exeFile.FullName)"
Write-Host "[SUCCESS] Found local installer: $($exeFile.FullName)"
Copy-Item -Path $exeFile.FullName -Destination "$env:TEMP\jan-installer.exe" -Force
Write-Host " Installer copied to: $env:TEMP\jan-installer.exe"
Write-Host "[SUCCESS] Installer copied to: $env:TEMP\jan-installer.exe"
# Don't set JAN_APP_PATH here - let the install script set it to the correct installed app path
echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $env:GITHUB_ENV
} else {
Write-Error " No .exe file found in artifact"
Write-Error "[FAILED] No .exe file found in artifact"
exit 1
}
} else {
Expand Down Expand Up @@ -196,9 +196,9 @@ jobs:
# Find the deb file in the artifact
DEB_FILE=$(find "${{ runner.temp }}/ubuntu-artifact" -name "*.deb" -type f | head -1)
if [ -n "$DEB_FILE" ]; then
echo " Found local installer: $DEB_FILE"
echo "[SUCCESS] Found local installer: $DEB_FILE"
cp "$DEB_FILE" "/tmp/jan-installer.deb"
echo " Installer copied to: /tmp/jan-installer.deb"
echo "[SUCCESS] Installer copied to: /tmp/jan-installer.deb"
echo "JAN_APP_PATH=/tmp/jan-installer.deb" >> $GITHUB_ENV
echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $GITHUB_ENV
if [ "${{ inputs.is_nightly }}" = "true" ]; then
Expand All @@ -207,7 +207,7 @@ jobs:
echo "JAN_PROCESS_NAME=Jan" >> $GITHUB_ENV
fi
else
echo " No .deb file found in artifact"
echo "[FAILED] No .deb file found in artifact"
exit 1
fi
else
Expand Down Expand Up @@ -296,9 +296,9 @@ jobs:
# Find the dmg file in the artifact
DMG_FILE=$(find "${{ runner.temp }}/macos-artifact" -name "*.dmg" -type f | head -1)
if [ -n "$DMG_FILE" ]; then
echo " Found local installer: $DMG_FILE"
echo "[SUCCESS] Found local installer: $DMG_FILE"
cp "$DMG_FILE" "/tmp/jan-installer.dmg"
echo " Installer copied to: /tmp/jan-installer.dmg"
echo "[SUCCESS] Installer copied to: /tmp/jan-installer.dmg"
echo "JAN_APP_PATH=/tmp/jan-installer.dmg" >> $GITHUB_ENV
echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $GITHUB_ENV
if [ "${{ inputs.is_nightly }}" = "true" ]; then
Expand All @@ -307,7 +307,7 @@ jobs:
echo "PROCESS_NAME=Jan" >> $GITHUB_ENV
fi
else
echo " No .dmg file found in artifact"
echo "[FAILED] No .dmg file found in artifact"
exit 1
fi
else
Expand Down Expand Up @@ -349,7 +349,7 @@ jobs:
else
echo "Homebrew not available, checking if tkinter works..."
python3 -c "import tkinter" || {
echo "⚠️ tkinter not available and Homebrew not found"
echo "[WARNING] tkinter not available and Homebrew not found"
echo "This may cause issues with mouse control"
}
fi
Expand All @@ -362,7 +362,7 @@ jobs:
echo "Installing Python dependencies..."
pip install --upgrade pip
pip install -r requirements.txt
echo " Python dependencies installed"
echo "[SUCCESS] Python dependencies installed"

- name: Setup ReportPortal environment
run: |
Expand Down
12 changes: 6 additions & 6 deletions autoqa/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -449,17 +449,17 @@ async def main():
# Update counters and log result
if test_passed:
test_results["passed"] += 1
logger.info(f" Test {i} PASSED: {test_data['path']}")
logger.info(f"[SUCCESS] Test {i} PASSED: {test_data['path']}")
else:
test_results["failed"] += 1
logger.error(f" Test {i} FAILED: {test_data['path']}")
logger.error(f"[FAILED] Test {i} FAILED: {test_data['path']}")

# Debug log for troubleshooting
logger.info(f"🔍 Debug - Test result: type={type(test_result)}, value={test_result}, success_field={test_result.get('success', 'N/A') if isinstance(test_result, dict) else 'N/A'}, final_passed={test_passed}")
logger.info(f"[INFO] Debug - Test result: type={type(test_result)}, value={test_result}, success_field={test_result.get('success', 'N/A') if isinstance(test_result, dict) else 'N/A'}, final_passed={test_passed}")

except Exception as e:
test_results["failed"] += 1
logger.error(f" Test {i} FAILED with exception: {test_data['path']} - {e}")
logger.error(f"[FAILED] Test {i} FAILED with exception: {test_data['path']} - {e}")

# Add delay between tests
if i < len(test_files):
Expand All @@ -477,10 +477,10 @@ async def main():
logger.info("=" * 50)

if test_results["failed"] > 0:
logger.error(f" Test execution completed with {test_results['failed']} failures!")
logger.error(f"[FAILED] Test execution completed with {test_results['failed']} failures!")
final_exit_code = 1
else:
logger.info(" All tests completed successfully!")
logger.info("[SUCCESS] All tests completed successfully!")
final_exit_code = 0

except KeyboardInterrupt:
Expand Down
14 changes: 7 additions & 7 deletions autoqa/reportportal_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ def upload_jan_logs(client, test_item_id, is_nightly=False, max_log_files=5):
client.log(
time=timestamp(),
level="WARNING",
message=f"📝 No Jan {app_type} application logs found",
message=f"[INFO] No Jan {app_type} application logs found",
item_id=test_item_id
)
return
Expand All @@ -236,7 +236,7 @@ def upload_jan_logs(client, test_item_id, is_nightly=False, max_log_files=5):
client.log(
time=timestamp(),
level="WARNING",
message=f"📝 Log file {file_name} skipped (size: {file_size} bytes > 50MB limit)",
message=f"[INFO] Log file {file_name} skipped (size: {file_size} bytes > 50MB limit)",
item_id=test_item_id
)
continue
Expand All @@ -251,7 +251,7 @@ def upload_jan_logs(client, test_item_id, is_nightly=False, max_log_files=5):
client.log(
time=timestamp(),
level="INFO",
message=f"📝 Jan {app_type} application log: {file_name}",
message=f"[INFO] Jan {app_type} application log: {file_name}",
item_id=test_item_id,
attachment={
"name": f"jan_{app_type}_log_{i}_{file_name}",
Expand All @@ -275,7 +275,7 @@ def upload_jan_logs(client, test_item_id, is_nightly=False, max_log_files=5):
client.log(
time=timestamp(),
level="INFO",
message=f"📝 Uploaded {len(log_files_to_upload)} Jan {app_type} log files (total available: {len(all_log_files)})",
message=f"[INFO] Uploaded {len(log_files_to_upload)} Jan {app_type} log files (total available: {len(all_log_files)})",
item_id=test_item_id
)

Expand Down Expand Up @@ -305,7 +305,7 @@ def upload_test_results_to_rp(client, launch_id, test_path, trajectory_dir, forc
client.log(
time=timestamp(),
level="ERROR",
message=" TEST FAILED \nNo trajectory directory found",
message="[FAILED] TEST FAILED [FAILED]\nNo trajectory directory found",
item_id=test_item_id
)

Expand Down Expand Up @@ -364,7 +364,7 @@ def upload_test_results_to_rp(client, launch_id, test_path, trajectory_dir, forc
if os.path.isdir(os.path.join(trajectory_dir, f)) and f.startswith("turn_")]

# Add clear status log
status_emoji = "" if final_status == "PASSED" else ""
status_emoji = "[SUCCESS]" if final_status == "PASSED" else "[FAILED]"
client.log(
time=timestamp(),
level="INFO" if final_status == "PASSED" else "ERROR",
Expand All @@ -383,7 +383,7 @@ def upload_test_results_to_rp(client, launch_id, test_path, trajectory_dir, forc
client.log(
time=timestamp(),
level="INFO",
message="🎥 Screen recording of test execution",
message="[INFO] Screen recording of test execution",
item_id=test_item_id,
attachment={
"name": f"test_recording_{formatted_test_path}.mp4",
Expand Down
4 changes: 2 additions & 2 deletions autoqa/scripts/macos_download.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ echo "Downloading Jan app from: $JAN_APP_URL"
curl -L -o "/tmp/jan-installer.dmg" "$JAN_APP_URL"

if [ ! -f "/tmp/jan-installer.dmg" ]; then
echo " Failed to download Jan app"
echo "[FAILED] Failed to download Jan app"
exit 1
fi

echo " Successfully downloaded Jan app"
echo "[SUCCESS] Successfully downloaded Jan app"
ls -la "/tmp/jan-installer.dmg"
12 changes: 6 additions & 6 deletions autoqa/scripts/macos_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ hdiutil attach "/tmp/jan-installer.dmg" -mountpoint "/tmp/jan-mount"
APP_FILE=$(find "/tmp/jan-mount" -name "*.app" -type d | head -1)

if [ -z "$APP_FILE" ]; then
echo " No .app file found in DMG"
echo "[Failed] No .app file found in DMG"
hdiutil detach "/tmp/jan-mount" || true
exit 1
fi
Expand Down Expand Up @@ -61,7 +61,7 @@ if [ -z "$APP_PATH" ]; then
fi

if [ -z "$APP_PATH" ]; then
echo " No executable found in MacOS folder"
echo "[FAILED] No executable found in MacOS folder"
ls -la "/Applications/$APP_NAME/Contents/MacOS/"
exit 1
fi
Expand All @@ -76,16 +76,16 @@ echo "Process name: $PROCESS_NAME"
echo "JAN_APP_PATH=$APP_PATH" >> $GITHUB_ENV
echo "PROCESS_NAME=$PROCESS_NAME" >> $GITHUB_ENV

echo " Waiting for Jan app first initialization (120 seconds)..."
echo "[INFO] Waiting for Jan app first initialization (120 seconds)..."
echo "This allows Jan to complete its initial setup and configuration"
sleep 120
echo " Initialization wait completed"
echo "[SUCCESS] Initialization wait completed"

# Verify installation
if [ -f "$APP_PATH" ]; then
echo " Jan app installed successfully"
echo "[SUCCESS] Jan app installed successfully"
ls -la "/Applications/$APP_NAME"
else
echo " Jan app installation failed - executable not found"
echo "[FAILED] Jan app installation failed - executable not found"
exit 1
fi
4 changes: 2 additions & 2 deletions autoqa/scripts/setup_permissions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Make all shell scripts executable
chmod +x "$SCRIPT_DIR"/*.sh

echo " All shell scripts are now executable:"
echo "[SUCCESS] All shell scripts are now executable:"
ls -la "$SCRIPT_DIR"/*.sh

echo " Permission setup completed"
echo "[SUCCESS] Permission setup completed"
4 changes: 2 additions & 2 deletions autoqa/scripts/ubuntu_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ sudo apt-get install -f -y
# Wait for installation to complete
sleep 10

echo " Waiting for Jan app first initialization (120 seconds)..."
echo "[INFO] Waiting for Jan app first initialization (120 seconds)..."
echo "This allows Jan to complete its initial setup and configuration"
sleep 120
echo " Initialization wait completed"
echo "[SUCCESS] Initialization wait completed"

# Verify installation based on nightly flag
if [ "$IS_NIGHTLY" = "true" ]; then
Expand Down
4 changes: 2 additions & 2 deletions autoqa/scripts/windows_install.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ catch {
# Wait a bit for installation to complete
Start-Sleep -Seconds 10

Write-Host " Waiting for Jan app first initialization (120 seconds)..."
Write-Host "[INFO] Waiting for Jan app first initialization (120 seconds)..."
Write-Host "This allows Jan to complete its initial setup and configuration"
Start-Sleep -Seconds 120
Write-Host " Initialization wait completed"
Write-Host "[SUCCESS] Initialization wait completed"

# Verify installation based on nightly flag
if ($isNightly) {
Expand Down
10 changes: 5 additions & 5 deletions autoqa/test_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ async def run_agent():
rp_client.log(
time=timestamp(),
level="INFO",
message="🎥 Screen recording of failed test",
message="[INFO] Screen recording of failed test",
item_id=test_item_id,
attachment={
"name": f"failed_test_recording_{formatted_test_path}.mp4",
Expand Down Expand Up @@ -298,9 +298,9 @@ async def run_agent():

if not enable_reportportal:
# Local development mode - log results
logger.info(f"🏠 LOCAL RESULT: {path} - {final_status} ({status_message})")
logger.info(f"📹 Video saved: {video_path}")
logger.info(f"📁 Trajectory: {trajectory_dir}")
logger.info(f"[INFO] LOCAL RESULT: {path} - {final_status} ({status_message})")
logger.info(f"[INFO] Video saved: {video_path}")
logger.info(f"[INFO] Trajectory: {trajectory_dir}")
else:
final_status = "FAILED"
status_message = "no trajectory found"
Expand All @@ -312,7 +312,7 @@ async def run_agent():
})

if not enable_reportportal:
logger.warning(f"🏠 LOCAL RESULT: {path} - {final_status} ({status_message})")
logger.warning(f"[INFO] LOCAL RESULT: {path} - {final_status} ({status_message})")

# Step 9: Always force close Jan app after test completion
logger.info(f"Cleaning up after test: {path}")
Expand Down
2 changes: 2 additions & 0 deletions autoqa/tests/new-user/1-user-start-chatting.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ Step-by-step instructions:
8. Wait for the model’s response.

If the model responds correctly, return: {"result": True}, otherwise return: {"result": False}.

In all your responses, use only plain ASCII characters. Do NOT use Unicode symbols
"""
Loading