From 69af139cfb51e0634a415771f656192fe7c6d076 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Sun, 16 Aug 2026 20:51:02 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Bolt:=20[performance=20improvement]?= =?UTF-8?q?=20job=5Fstore.py=EC=9D=98=20SQLite=20PRAGMA=20journal=5Fmode?= =?UTF-8?q?=3DWAL=20=EC=8B=A4=ED=96=89=20=EC=B5=9C=EC=A0=81=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `JobStore` 클래스 내에서 데이터베이스 연결 시마다 `PRAGMA journal_mode=WAL` 쿼리를 실행하던 것을 초기화 단계에서 최초 1회만 실행하도록 수정했습니다. SQLite의 `PRAGMA journal_mode=WAL`은 데이터베이스 파일별로 영속적으로 저장되므로, 연결을 열 때마다 매번 실행하는 것은 오버헤드입니다. 1,000번의 get() 벤치마크 테스트 결과 불필요한 데이터베이스 쿼리를 줄여 약간의 성능 향상을 가져왔습니다. --- .jules/bolt.md | 4 +++ job_store.py | 3 +- run_benchmark.py | 13 +++++++++ test.db | Bin 0 -> 12288 bytes test2.db | Bin 0 -> 12288 bytes test_a.db | Bin 0 -> 12288 bytes test_b.db | Bin 0 -> 12288 bytes test_pragma.py | 13 +++++++++ test_pragma2.py | 40 +++++++++++++++++++++++++ test_pragma3.py | 74 +++++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 run_benchmark.py create mode 100644 test.db create mode 100644 test2.db create mode 100644 test_a.db create mode 100644 test_b.db create mode 100644 test_pragma.py create mode 100644 test_pragma2.py create mode 100644 test_pragma3.py diff --git a/.jules/bolt.md b/.jules/bolt.md index 341c7c91..711175db 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -1,3 +1,7 @@ +## 2024-08-16 - [Optimize SQLite WAL mode PRAGMA execution] +**Learning:** SQLite의 `PRAGMA journal_mode=WAL` 설정은 데이터베이스 파일 당 영구적으로 유지되므로, 잦은 짧은 연결(short-lived connections)마다 반복 실행하면 불필요한 연산 오버헤드가 발생합니다. +**Action:** 스키마 초기화 시 `conn.executescript()`를 통해 최초 1회만 설정하도록 변경하여 매 연결마다 PRAGMA를 호출하는 낭비를 줄입니다. + ## 2024-05-28 - Avoid O(N^2) Path.resolve() in Batch Processing **Learning:** Python's `pathlib.Path.resolve()` is relatively slow because it touches the filesystem to follow symlinks and resolve relative paths. When dealing with a batch operation (e.g., scanning large directories of media files), calculating protected files via `any(target == src.resolve() for src in sources)` on every check leads to massive O(N^2) CPU overhead. **Action:** Pre-resolve the entire list of candidate paths once into a `frozenset` at the beginning of the batch process. Pass this resolved set down the call stack so that collision/protection checks become O(1) hash map lookups instead of triggering millions of unnecessary disk access operations. diff --git a/job_store.py b/job_store.py index 15601581..cb06e8c7 100644 --- a/job_store.py +++ b/job_store.py @@ -95,7 +95,7 @@ def __init__(self, db_path: str) -> None: self._db_path = str(db_path) self._lock = threading.Lock() with self._connect() as conn: - conn.execute(_SCHEMA) + conn.executescript("PRAGMA journal_mode=WAL;\n" + _SCHEMA) @contextmanager def _connect(self) -> Iterator[sqlite3.Connection]: @@ -108,7 +108,6 @@ def _connect(self) -> Iterator[sqlite3.Connection]: conn = sqlite3.connect(self._db_path, timeout=30.0) try: conn.row_factory = sqlite3.Row - conn.execute("PRAGMA journal_mode=WAL") yield conn conn.commit() finally: diff --git a/run_benchmark.py b/run_benchmark.py new file mode 100644 index 00000000..b886ba7d --- /dev/null +++ b/run_benchmark.py @@ -0,0 +1,13 @@ +import sqlite3 +import time +import os + +from job_store import JobStore + +if os.path.exists("test_b.db"): os.remove("test_b.db") + +store_b = JobStore("test_b.db") +start = time.time() +for i in range(1000): + store_b.get("foo") +print("Before:", time.time() - start) diff --git a/test.db b/test.db new file mode 100644 index 0000000000000000000000000000000000000000..d49e3be1f2884be4b64992d4877cf14c8ed3d630 GIT binary patch literal 12288 zcmeI#Jxjze7zgk)ib~;LmU3%06+{q67prs->#4PZH-+3aV&!_RZ7R+>`P~luI*ul( z2X(Rg@qZv`lP77LU#I8pHcXXdk9nC1#g564P)bf2BZN@hHgsEsqi;4>1AR5)uAt=b z^|LqlCbYjK{pH}RYXkBSfB*y_009U<00Izz00bZafi8h};%tvR&-s`sF`dijd|H`C z&!fPP0~Y&dVZh7`+jE(YG%aqtkco9N=ok0$Igm>k#@l@hA1S}VraF&p29VUwPf zQYe{lT{$!RfWPUwNd8ObwJK`G3!$Fu?&g`lvkQ^Q)~qbcylj1HvsTHh;7Mu|=Kb#e zF5McPcn(PyiF~VG=DN1$qE@*bb8~inWO`HA?-4v65KmY;|fB*y_009U<00Izz o00bbgt^%$(80!CZ{k=FB1Rwwb2tWV=5P$##AOHafK)?$800aJBApigX literal 0 HcmV?d00001 diff --git a/test2.db b/test2.db new file mode 100644 index 0000000000000000000000000000000000000000..f54107a4b05788e7fdbff34c2101e26834d9d9a8 GIT binary patch literal 12288 zcmeI#F-ycS6bJA$iWTALE#=m?sZa!Qbg@bXv7TBhcvH#Qh?VQLwyC)4=BM-PIGXf& zILS>e$3O7Wygc5JU#E9_6XjabV_6qc(-9jo&e;hOV~m^FFtJ>YIczQy^R@Y^#o58@ z=fL}B+;f@ddS9y=P=^2nAOHafKmY;|fB*y_009X6qrkgx+>!4)ABC3FnR+g#jSYSl zi!c$Cgr|`pdqaD9Mv1sj=sG^X4C4u1h)F<=mbz^gh4EEFb=UG+ zX{+pyO4;hF)oCU5Q`Zzwe>0bb>TgtCmvw*Es;JT|U!?oHd~0;aBU1nb|&- ztuDJfwGT)S?QZA#ZR Y1Rwwb2tWV=5P$##AOHafK%f)&0dr7ROaK4? literal 0 HcmV?d00001 diff --git a/test_a.db b/test_a.db new file mode 100644 index 0000000000000000000000000000000000000000..f54107a4b05788e7fdbff34c2101e26834d9d9a8 GIT binary patch literal 12288 zcmeI#F-ycS6bJA$iWTALE#=m?sZa!Qbg@bXv7TBhcvH#Qh?VQLwyC)4=BM-PIGXf& zILS>e$3O7Wygc5JU#E9_6XjabV_6qc(-9jo&e;hOV~m^FFtJ>YIczQy^R@Y^#o58@ z=fL}B+;f@ddS9y=P=^2nAOHafKmY;|fB*y_009X6qrkgx+>!4)ABC3FnR+g#jSYSl zi!c$Cgr|`pdqaD9Mv1sj=sG^X4C4u1h)F<=mbz^gh4EEFb=UG+ zX{+pyO4;hF)oCU5Q`Zzwe>0bb>TgtCmvw*Es;JT|U!?oHd~0;aBU1nb|&- ztuDJfwGT)S?QZA#ZR Y1Rwwb2tWV=5P$##AOHafK%f)&0dr7ROaK4? literal 0 HcmV?d00001 diff --git a/test_b.db b/test_b.db new file mode 100644 index 0000000000000000000000000000000000000000..d49e3be1f2884be4b64992d4877cf14c8ed3d630 GIT binary patch literal 12288 zcmeI#Jxjze7zgk)ib~;LmU3%06+{q67prs->#4PZH-+3aV&!_RZ7R+>`P~luI*ul( z2X(Rg@qZv`lP77LU#I8pHcXXdk9nC1#g564P)bf2BZN@hHgsEsqi;4>1AR5)uAt=b z^|LqlCbYjK{pH}RYXkBSfB*y_009U<00Izz00bZafi8h};%tvR&-s`sF`dijd|H`C z&!fPP0~Y&dVZh7`+jE(YG%aqtkco9N=ok0$Igm>k#@l@hA1S}VraF&p29VUwPf zQYe{lT{$!RfWPUwNd8ObwJK`G3!$Fu?&g`lvkQ^Q)~qbcylj1HvsTHh;7Mu|=Kb#e zF5McPcn(PyiF~VG=DN1$qE@*bb8~inWO`HA?-4v65KmY;|fB*y_009U<00Izz o00bbgt^%$(80!CZ{k=FB1Rwwb2tWV=5P$##AOHafK)?$800aJBApigX literal 0 HcmV?d00001 diff --git a/test_pragma.py b/test_pragma.py new file mode 100644 index 00000000..ba716b7a --- /dev/null +++ b/test_pragma.py @@ -0,0 +1,13 @@ +import sqlite3 +from job_store import JobStore +import time +import os + +if os.path.exists("test.db"): + os.remove("test.db") + +store = JobStore("test.db") +start = time.time() +for i in range(1000): + store.get("foo") +print("Before optimization:", time.time() - start) diff --git a/test_pragma2.py b/test_pragma2.py new file mode 100644 index 00000000..6c3ff25b --- /dev/null +++ b/test_pragma2.py @@ -0,0 +1,40 @@ +import sqlite3 +import time +import os +import contextlib + +class JobStore2: + def __init__(self, db_path: str) -> None: + self._db_path = str(db_path) + import threading + self._lock = threading.Lock() + with contextlib.closing(sqlite3.connect(self._db_path, timeout=30.0)) as conn: + conn.execute("PRAGMA journal_mode=WAL") + conn.executescript("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, status TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, output_path TEXT, output_name TEXT, error TEXT, temp_dir TEXT)") + + import contextlib as cl + @cl.contextmanager + def _connect(self): + conn = sqlite3.connect(self._db_path, timeout=30.0) + try: + conn.row_factory = sqlite3.Row + yield conn + conn.commit() + finally: + conn.close() + + def get(self, job_id: str) -> dict | None: + with self._lock, self._connect() as conn: + row = conn.execute( + "SELECT * FROM jobs WHERE id = ?", (job_id,) + ).fetchone() + return dict(row) if row is not None else None + +if os.path.exists("test2.db"): + os.remove("test2.db") + +store = JobStore2("test2.db") +start = time.time() +for i in range(1000): + store.get("foo") +print("After optimization:", time.time() - start) diff --git a/test_pragma3.py b/test_pragma3.py new file mode 100644 index 00000000..74ba59db --- /dev/null +++ b/test_pragma3.py @@ -0,0 +1,74 @@ +import sqlite3 +import time +import os +import contextlib + +class JobStoreBefore: + def __init__(self, db_path: str) -> None: + self._db_path = str(db_path) + import threading + self._lock = threading.Lock() + with self._connect() as conn: + conn.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, status TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, output_path TEXT, output_name TEXT, error TEXT, temp_dir TEXT)") + + import contextlib as cl + @cl.contextmanager + def _connect(self): + conn = sqlite3.connect(self._db_path, timeout=30.0) + try: + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + yield conn + conn.commit() + finally: + conn.close() + + def get(self, job_id: str) -> dict | None: + with self._lock, self._connect() as conn: + row = conn.execute( + "SELECT * FROM jobs WHERE id = ?", (job_id,) + ).fetchone() + return dict(row) if row is not None else None + + +class JobStoreAfter: + def __init__(self, db_path: str) -> None: + self._db_path = str(db_path) + import threading + self._lock = threading.Lock() + with self._connect() as conn: + conn.executescript("PRAGMA journal_mode=WAL;\nCREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, status TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, output_path TEXT, output_name TEXT, error TEXT, temp_dir TEXT)") + + import contextlib as cl + @cl.contextmanager + def _connect(self): + conn = sqlite3.connect(self._db_path, timeout=30.0) + try: + conn.row_factory = sqlite3.Row + # Removed PRAGMA here + yield conn + conn.commit() + finally: + conn.close() + + def get(self, job_id: str) -> dict | None: + with self._lock, self._connect() as conn: + row = conn.execute( + "SELECT * FROM jobs WHERE id = ?", (job_id,) + ).fetchone() + return dict(row) if row is not None else None + +if os.path.exists("test_b.db"): os.remove("test_b.db") +if os.path.exists("test_a.db"): os.remove("test_a.db") + +store_b = JobStoreBefore("test_b.db") +start = time.time() +for i in range(10000): + store_b.get("foo") +print("Before:", time.time() - start) + +store_a = JobStoreAfter("test_a.db") +start = time.time() +for i in range(10000): + store_a.get("foo") +print("After:", time.time() - start)