Skip to content

Commit

Permalink
Add in-client views counter to stats pages
Browse files Browse the repository at this point in the history
  • Loading branch information
rubenwardy committed Oct 28, 2024
1 parent 2ff11de commit 39cf1af
Show file tree
Hide file tree
Showing 11 changed files with 154 additions and 8 deletions.
11 changes: 10 additions & 1 deletion app/blueprints/api/endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from app.markdown import render_markdown
from app.models import Tag, PackageState, PackageType, Package, db, PackageRelease, Permission, \
MinetestRelease, APIToken, PackageScreenshot, License, ContentWarning, User, PackageReview, Thread, Collection, \
PackageAlias, Language
PackageAlias, Language, PackageDailyStats
from app.querybuilder import QueryBuilder
from app.utils import is_package_page, get_int_or_abort, url_set_query, abs_url, is_yes, get_request_date, cached, \
cors_allowed
Expand All @@ -39,6 +39,7 @@
from .auth import is_api_authd
from .support import error, api_create_vcs_release, api_create_zip_release, api_create_screenshot, \
api_order_screenshots, api_edit_package, api_set_cover_image
from ...rediscache import make_view_key, set_temp_key, has_key


@bp.route("/api/packages/")
Expand Down Expand Up @@ -99,6 +100,14 @@ def package_view(package):
@is_package_page
@cors_allowed
def package_view_client(package: Package):
ip = request.headers.get("X-Forwarded-For") or request.remote_addr
if ip is not None and (request.headers.get("User-Agent") or "").startswith("Minetest"):
key = make_view_key(ip, package)
if not has_key(key):
set_temp_key(key, "true")
PackageDailyStats.notify_view(package)
db.session.commit()

protocol_version = request.args.get("protocol_version")
engine_version = request.args.get("engine_version")
if protocol_version or engine_version:
Expand Down
7 changes: 5 additions & 2 deletions app/blueprints/packages/releases.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

import os

from flask import render_template, request, redirect, flash, url_for, abort
Expand All @@ -31,6 +32,7 @@
from app.tasks.importtasks import check_update_config
from app.utils import is_user_bot, is_package_page, nonempty_or_none, normalize_line_endings
from . import bp, get_package_tabs
from app.utils.version import is_minetest_v510


@bp.route("/packages/<author>/<name>/releases/", methods=["GET", "POST"])
Expand Down Expand Up @@ -127,9 +129,10 @@ def download_release(package, id):

ip = request.headers.get("X-Forwarded-For") or request.remote_addr
if ip is not None and not is_user_bot():
is_minetest = (request.headers.get("User-Agent") or "").startswith("Minetest")
is_minetest = (request.headers.get("User-Agent") or "").startswith("Minetest/")
is_v510 = is_minetest and is_minetest_v510(request.headers.get("User-Agent"))
reason = request.args.get("reason")
PackageDailyStats.update(package, is_minetest, reason)
PackageDailyStats.notify_download(package, is_minetest, is_v510, reason)

key = make_download_key(ip, release.package)
if not has_key(key):
Expand Down
2 changes: 2 additions & 0 deletions app/flatpages/help/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ curl -X DELETE https://content.minetest.net/api/delete-token/ \
* `reason_new`: list of integers per day.
* `reason_dependency`: list of integers per day.
* `reason_update`: list of integers per day.
* `views_minetest`: list of integers per day.
* GET `/api/package_stats/`
* Returns last 30 days of daily stats for _all_ packages.
* An object with the following keys:
Expand Down Expand Up @@ -437,6 +438,7 @@ Example:
* `reason_new`: list of integers per day.
* `reason_dependency`: list of integers per day.
* `reason_update`: list of integers per day.
* `views_minetest`: list of integers per day.


## Topics
Expand Down
5 changes: 3 additions & 2 deletions app/logic/graphs.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def daterange(start_date, end_date):


keys = ["platform_minetest", "platform_other", "reason_new",
"reason_dependency", "reason_update"]
"reason_dependency", "reason_update", "views_minetest"]


def flatten_data(stats):
Expand Down Expand Up @@ -78,7 +78,8 @@ def get_package_stats_for_user(user: User, start_date: Optional[datetime.date],
func.sum(PackageDailyStats.platform_other).label("platform_other"),
func.sum(PackageDailyStats.reason_new).label("reason_new"),
func.sum(PackageDailyStats.reason_dependency).label("reason_dependency"),
func.sum(PackageDailyStats.reason_update).label("reason_update")) \
func.sum(PackageDailyStats.reason_update).label("reason_update"),
func.sum(PackageDailyStats.views_minetest).label("views_minetest")) \
.filter(PackageDailyStats.package.has(author_id=user.id))

if start_date:
Expand Down
25 changes: 24 additions & 1 deletion app/models/packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -1437,8 +1437,11 @@ class PackageDailyStats(db.Model):
reason_dependency = db.Column(db.Integer, nullable=False, default=0)
reason_update = db.Column(db.Integer, nullable=False, default=0)

views_minetest = db.Column(db.Integer, nullable=False, default=0)
v510 = db.Column(db.Integer, nullable=False, default=0)

@staticmethod
def update(package: Package, is_minetest: bool, reason: str):
def notify_download(package: Package, is_minetest: bool, is_v510: bool, reason: str):
date = datetime.datetime.utcnow().date()

to_update = dict()
Expand All @@ -1462,6 +1465,26 @@ def update(package: Package, is_minetest: bool, reason: str):
to_update[field_reason] = getattr(PackageDailyStats, field_reason) + 1
kwargs[field_reason] = 1

if is_v510:
to_update["v510"] = PackageDailyStats.v510 + 1
kwargs["v510"] = 1

stmt = insert(PackageDailyStats).values(**kwargs)
stmt = stmt.on_conflict_do_update(
index_elements=[PackageDailyStats.package_id, PackageDailyStats.date],
set_=to_update
)

conn = db.session.connection()
conn.execute(stmt)

@staticmethod
def notify_view(package: Package):
date = datetime.datetime.utcnow().date()

to_update = {"views_minetest": PackageDailyStats.views_minetest + 1}
kwargs = {"package_id": package.id, "date": date, "views_minetest": 1}

stmt = insert(PackageDailyStats).values(**kwargs)
stmt = stmt.on_conflict_do_update(
index_elements=[PackageDailyStats.package_id, PackageDailyStats.date],
Expand Down
10 changes: 10 additions & 0 deletions app/public/static/js/package_charts.js
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,16 @@ async function load_data() {
};
new Chart(ctx, config);
}

{
const ctx = document.getElementById("chart-views").getContext("2d");
const data = {
datasets: [
{ label: "Luanti", data: getData(json.views_minetest) },
],
};
setup_chart(ctx, data, annotations);
}
}


Expand Down
7 changes: 6 additions & 1 deletion app/rediscache.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.

from . import redis_client
from .models import Package

# This file acts as a facade between the rest of the code and redis,
# and also means that the rest of the code avoids knowing about `app`
Expand All @@ -23,10 +24,14 @@
EXPIRY_TIME_S = 2*7*24*60*60 # 2 weeks


def make_download_key(ip, package):
def make_download_key(ip: str, package: Package):
return "{}/{}/{}".format(ip, package.author.username, package.name)


def make_view_key(ip: str, package: Package):
return "view/{}/{}/{}".format(ip, package.author.username, package.name)


def set_temp_key(key, v):
redis_client.set(key, v, ex=EXPIRY_TIME_S)

Expand Down
8 changes: 7 additions & 1 deletion app/templates/macros/stats.html
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<script src="/static/libs/chart.min.js"></script>
<script src="/static/libs/chartjs-adapter-date-fns.bundle.min.js"></script>
<script src="/static/libs/chartjs-plugin-annotation.min.js"></script>
<script src="/static/js/package_charts.js?v=2"></script>
<script src="/static/js/package_charts.js?v=3"></script>
{% endmacro %}


Expand Down Expand Up @@ -118,6 +118,12 @@ <h3 class="mt-5">{{ _("Downloads by Reason") }}</h3>
</div>
</div>

<h3 class="mt-5">{{ _("Views inside Luanti") }}</h3>
<p>
{{ _("Number of package page views inside the Luanti client. v5.10 and later only.") }}
</p>
<canvas id="chart-views" class="chart"></canvas>

<h3 style="margin-top: 6em;">{{ _("Need more stats?") }}</h3>
<p>
{{ _("Check out the ContentDB Grafana dashboard for CDB-wide stats") }}
Expand Down
30 changes: 30 additions & 0 deletions app/tests/unit/utils/test_version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# ContentDB
# Copyright (C) rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

from app.utils.version import is_minetest_v510


def test_is_minetest_v510():
assert not is_minetest_v510("Minetest/5.9.1 (Windows/10.0.22621 x86_64)")
assert not is_minetest_v510("Minetest/")
assert not is_minetest_v510("Minetest/5.9.1")

assert is_minetest_v510("Minetest/5.10.0")
assert is_minetest_v510("Minetest/5.10.1")
assert is_minetest_v510("Minetest/5.11.0")
assert is_minetest_v510("Minetest/5.10")

assert not is_minetest_v510("Minetest/6.12")
29 changes: 29 additions & 0 deletions app/utils/version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# ContentDB
# Copyright (C) rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.


def is_minetest_v510(user_agent: str) -> bool:
parts = user_agent.split(" ")
version = parts[0].split("/")[1]
try:
digits = list(map(lambda x: int(x), version.split(".")))
except ValueError:
return False

if len(digits) < 2:
return False

return digits[0] == 5 and digits[1] >= 10
28 changes: 28 additions & 0 deletions migrations/versions/d52f6901b707_.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
"""empty message
Revision ID: d52f6901b707
Revises: daa040b727b2
Create Date: 2024-10-22 21:18:23.929298
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = 'd52f6901b707'
down_revision = 'daa040b727b2'
branch_labels = None
depends_on = None


def upgrade():
with op.batch_alter_table('package_daily_stats', schema=None) as batch_op:
batch_op.add_column(sa.Column('views_minetest', sa.Integer(), nullable=False, server_default="0"))
batch_op.add_column(sa.Column('v510', sa.Integer(), nullable=False, server_default="0"))


def downgrade():
with op.batch_alter_table('package_daily_stats', schema=None) as batch_op:
batch_op.drop_column('views_minetest')
batch_op.drop_column('v510')

0 comments on commit 39cf1af

Please sign in to comment.