From 468eff7bcf8711eb3efc79d97a16a2a24e61df53 Mon Sep 17 00:00:00 2001 From: jueri Date: Thu, 5 Dec 2024 09:13:23 +0100 Subject: [PATCH] make calls to base and experimental systems concurrently --- web/app/api/rankings.py | 3 +- web/app/db_async.py | 7 ++- web/app/services/ranking_service.py | 81 +++++++++++++++-------------- web/requirements.txt | 3 +- 4 files changed, 52 insertions(+), 42 deletions(-) diff --git a/web/app/api/rankings.py b/web/app/api/rankings.py index 7b89460..962076e 100644 --- a/web/app/api/rankings.py +++ b/web/app/api/rankings.py @@ -4,7 +4,7 @@ from app.services.session_service import create_new_session from app.services.system_service import get_least_served_system from app.services.profile_service import profile_route -from flask import jsonify, request, current_app +from flask import jsonify, request from pytz import timezone import asyncio @@ -75,7 +75,6 @@ def ranking_from_db(rid): @api.route("/ranking", methods=["GET"]) -@profile_route def ranking(): """Produce a ranking for current session diff --git a/web/app/db_async.py b/web/app/db_async.py index 81e3267..8546042 100644 --- a/web/app/db_async.py +++ b/web/app/db_async.py @@ -7,6 +7,11 @@ def init_async_session(app): database_uri = app.config["SQLALCHEMY_DATABASE_URI"] database_uri = database_uri.replace("postgresql", "postgresql+asyncpg") + # database_uri = database_uri.replace("sqlite", "sqlite+aiosqlite") + if app.config["TESTING"]: + database_uri = database_uri.replace("sqlite:///", "sqlite+aiosqlite:///") + + print(f"Using database URI: {database_uri}") engine = create_async_engine(database_uri, echo=False) @@ -14,4 +19,4 @@ def init_async_session(app): bind=engine, class_=AsyncSession, expire_on_commit=False ) app.extensions["async_session"] = async_session_factory - print(f"Async session initialized: {async_session}") + print(f"Async session initialized: {async_session_factory}") diff --git a/web/app/services/ranking_service.py b/web/app/services/ranking_service.py index 927a3c5..f495717 100644 --- a/web/app/services/ranking_service.py +++ b/web/app/services/ranking_service.py @@ -6,7 +6,7 @@ import docker from app.extensions import cache -from app.models import Result, System, db +from app.models import Result, System from app.services.interleave_service import interleave_rankings from flask import current_app from pytz import timezone @@ -38,9 +38,35 @@ async def request_results_from_conatiner(container_name, query, rpp, page): return content.json() +def extract_hits(result, container_name, type): + # Extract hits list from result and allow custom fields for docid and hits_path + docid_name = current_app.config["SYSTEMS_CONFIG"][container_name].get( + "docid", "docid" + ) + hits_path = current_app.config["SYSTEMS_CONFIG"][container_name].get("hits_path") + + if hits_path is None: + hits = result["itemlist"] + else: + hits = hits_path.find(result)[0].value + + if isinstance(hits[0], dict): + item_dict = { + i + 1: {"docid": hits[i][docid_name], "type": type} + for i in range(0, len(hits)) + } + else: + item_dict = { + i + 1: {"docid": hits[i], "type": type} for i in range(0, len(hits)) + } + + return item_dict, hits + + async def query_system(container_name, query, rpp, page, session_id, type="EXP"): """query a system with a given query and return the ranking and the result""" - current_app.logger.debug(f'Produce ranking with system: "{container_name}"...') + current_app.logger.debug(f'Produce ranking with system: "{container_name}"') + q_date = datetime.now(tz).replace(tzinfo=None, microsecond=0) ts_start = time.time() @@ -60,52 +86,33 @@ async def query_system(container_name, query, rpp, page, session_id, type="EXP") await session.commit() result = await request_results_from_conatiner(container_name, query, rpp, page) + item_dict, hits = extract_hits(result, container_name, type) # calc query execution time in ms ts_end = time.time() q_time = round((ts_end - ts_start) * 1000) - # Extract hits list from result and allow custom fields for docid and hits_path - docid_name = current_app.config["SYSTEMS_CONFIG"][container_name].get( - "docid", "docid" - ) - hits_path = current_app.config["SYSTEMS_CONFIG"][container_name].get("hits_path") - - if hits_path is None: - hits = result["itemlist"] - else: - hits = hits_path.find(result)[0].value - - if isinstance(hits[0], dict): - item_dict = { - i + 1: {"docid": hits[i][docid_name], "type": type} - for i in range(0, len(hits)) - } - else: - item_dict = { - i + 1: {"docid": hits[i], "type": type} for i in range(0, len(hits)) - } - + # Save the ranking to the database async with async_session_factory() as session: - # Save the ranking to the database system_id = await session.execute( select(System).where(System.name == container_name) ) system_id = system_id.scalar() + ranking = Result( + session_id=session_id, + system_id=system_id.id, + type="RANK", + q=query, + q_date=q_date, + q_time=q_time, + num_found=len(hits), + page=page, + rpp=rpp, + items=item_dict, + ) + async with async_session_factory() as session: - ranking = Result( - session_id=session_id, - system_id=system_id.id, - type="RANK", - q=query, - q_date=q_date, - q_time=q_time, - num_found=len(hits), - page=page, - rpp=rpp, - items=item_dict, - ) session.add(ranking) await session.commit() @@ -190,9 +197,7 @@ def build_simple_response(ranking_obj): # @cache.memoize(timeout=600) async def make_ranking(container_name, query, rpp, page, session_id): - if current_app.config["INTERLEAVE"]: - current_app.logger.warning("Started gathering") baseline, experimental = await asyncio.gather( query_system( diff --git a/web/requirements.txt b/web/requirements.txt index 4872d3e..b32debe 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -86,4 +86,5 @@ Flask-APScheduler==1.13.1 requests-mock==1.12.1 jsonpath-ng==1.6.1 Flask-Caching==2.3.0 -httpx \ No newline at end of file +httpx +aiosqlite \ No newline at end of file