Skip to content

feat: Use both RssAnon and VmRSS in CI benchmarks #133

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 18, 2024
Merged

feat: Use both RssAnon and VmRSS in CI benchmarks #133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
03af5a8
feat: Use RssAnon instead of VmRSS
KShivendu Apr 18, 2024
2cba673
feat: Show both RSSAnon and VMRss
KShivendu Apr 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions tools/qdrant_collect_stats.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@ BENCH_SERVER_NAME=${SERVER_NAME:-"benchmark-server-1"}

IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_SERVER_NAME")

MEMORY_USAGE=$(ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "grep VmRSS /proc/\$(pidof qdrant)/status | awk '{print \$2}'")
VM_RSS_MEMORY_USAGE=$(ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "grep VmRSS /proc/\$(pidof qdrant)/status | awk '{print \$2}'")
RSS_ANON_MEMORY_USAGE=$(ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "grep RssAnon /proc/\$(pidof qdrant)/status | awk '{print \$2}'")

CURRENT_DATE=$(date +%Y-%m-%d-%H-%M-%S)

echo "$MEMORY_USAGE" > results/memory-usage-"${CURRENT_DATE}".txt
echo "$VM_RSS_MEMORY_USAGE" > results/vm-rss-memory-usage-"${CURRENT_DATE}".txt
echo "$RSS_ANON_MEMORY_USAGE" > results/rss-anon-memory-usage-"${CURRENT_DATE}".txt

ROOT_API_RESPONSE=$(ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "curl -s http://localhost:6333/")

Expand Down
3 changes: 2 additions & 1 deletion tools/run_ci.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ bash -x "${SCRIPT_PATH}/run_remote_benchmark.sh"
# -t sorts by modification time
export SEARCH_RESULTS_FILE=$(ls -t results/*-search-*.json | head -n 1)
export UPLOAD_RESULTS_FILE=$(ls -t results/*-upload-*.json | head -n 1)
export MEMORY_USAGE_FILE=$(ls -t results/memory-usage-*.txt | head -n 1)
export VM_RSS_MEMORY_USAGE_FILE=$(ls -t results/vm-rss-memory-usage-*.txt | head -n 1)
export RSS_ANON_MEMORY_USAGE_FILE=$(ls -t results/rss-anon-memory-usage-*.txt | head -n 1)
export ROOT_API_RESPONSE_FILE=$(ls -t results/root-api-*.json | head -n 1)

bash -x "${SCRIPT_PATH}/upload_results_postgres.sh"
3 changes: 2 additions & 1 deletion tools/run_client_script.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ ssh -tt "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "${RUN_EXPERIMENT}"

SEARCH_RESULT_FILE=$(ssh "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "ls -t results/*-search-*.json | head -n 1")
UPLOAD_RESULT_FILE=$(ssh "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "ls -t results/*-upload-*.json | head -n 1")
MEMORY_USAGE_FILE=$(ssh "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "ls -t results/memory-usage-*.txt | head -n 1")
VM_RSS_MEMORY_USAGE_FILE=$(ssh "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "ls -t results/vm-rss-memory-usage-*.txt | head -n 1")
RSS_ANON_MEMORY_USAGE_FILE=$(ssh "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "ls -t results/rss-anon-memory-usage-*.txt | head -n 1")

mkdir -p results

Expand Down
24 changes: 16 additions & 8 deletions tools/upload_results_postgres.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
# id SERIAL PRIMARY key,
# engine VARCHAR(255),
# branch VARCHAR(255),
# commit CHAR(40),
# commit CHAR(40),
# dataset VARCHAR(255),
# measure_timestamp TIMESTAMP,
# upload_time real,
Expand All @@ -17,12 +17,14 @@
# mean_precisions real,
# p95_time real,
# p99_time real,
# memory_usage real
# vm_rss_mem real,
# rss_anon_mem real
# );

SEARCH_RESULTS_FILE=${SEARCH_RESULTS_FILE:-""}
UPLOAD_RESULTS_FILE=${UPLOAD_RESULTS_FILE:-""}
MEMORY_USAGE_FILE=${MEMORY_USAGE_FILE:-""}
VM_RSS_MEMORY_USAGE_FILE=${VM_RSS_MEMORY_USAGE_FILE:-""}
RSS_ANON_MEMORY_USAGE_FILE=${RSS_ANON_MEMORY_USAGE_FILE:-""}
ROOT_API_RESPONSE_FILE=${ROOT_API_RESPONSE_FILE:-""}
POSTGRES_TABLE=${POSTGRES_TABLE:-"benchmark"}

Expand All @@ -41,8 +43,13 @@ if [[ -z "$UPLOAD_RESULTS_FILE" ]]; then
fi


if [[ -z "$MEMORY_USAGE_FILE" ]]; then
echo "MEMORY_USAGE_FILE is not set"
if [[ -z "$VM_RSS_MEMORY_USAGE_FILE" ]]; then
echo "VM_RSS_MEMORY_USAGE_FILE is not set"
exit 1
fi

if [[ -z "$RSS_ANON_MEMORY_USAGE_FILE" ]]; then
echo "RSS_ANON_MEMORY_USAGE_FILEis not set"
exit 1
fi

Expand All @@ -59,15 +66,16 @@ P99_TIME=$(jq -r '.results.p99_time' "$SEARCH_RESULTS_FILE")
UPLOAD_TIME=$(jq -r '.results.upload_time' "$UPLOAD_RESULTS_FILE")
INDEXING_TIME=$(jq -r '.results.total_time' "$UPLOAD_RESULTS_FILE")

MEMORY_USAGE=$(cat "$MEMORY_USAGE_FILE")
VM_RSS_MEMORY_USAGE=$(cat "$VM_RSS_MEMORY_USAGE_FILE")
RSS_ANON_MEMORY_USAGE=$(cat "$RSS_ANON_MEMORY_USAGE_FILE")

QDRANT_COMMIT=$(jq -r '.commit' "$ROOT_API_RESPONSE_FILE")

MEASURE_TIMESTAMP=${MEASURE_TIMESTAMP:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")}


docker run --rm jbergknoff/postgresql-client "postgresql://qdrant:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/postgres" -c "
INSERT INTO ${POSTGRES_TABLE} (engine, branch, commit, dataset, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage)
VALUES ('qdrant-ci', '${QDRANT_VERSION}', '${QDRANT_COMMIT}', '${DATASETS}', '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE});
INSERT INTO ${POSTGRES_TABLE} (engine, branch, commit, dataset, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, vm_rss_mem, rss_anon_mem)
VALUES ('qdrant-ci', '${QDRANT_VERSION}', '${QDRANT_COMMIT}', '${DATASETS}', '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${VM_RSS_MEMORY_USAGE}, ${RSS_ANON_MEMORY_USAGE});
"