chore: Fix sqlite3 deprecation. Ignore pkg_resources deprecation #328

Author: jpmckinney

.github/workflows/tests.yml

@@ -16,7 +16,13 @@ jobs:
           cache: pip
           cache-dependency-path: setup.py
       - run: pip install -e .[test]
-      - run: pytest -W error -W ignore::ResourceWarning tests --cov scrapyd
+      # Python 3.12 deprecates pkg_resources (also used by py-html-checker).
+      - run: |
+          pytest -W error -W ignore::ResourceWarning \
+            -W ignore::DeprecationWarning:scrapyd.runner \
+            -W ignore::DeprecationWarning:html_checker \
+            -W ignore::DeprecationWarning:pkg_resources \
+            tests --cov scrapyd
       # Occasional "ConnectionRefusedError: [Errno 111] Connection refused".
       - name: Run integration tests
         run: |

scrapyd/launcher.py

@@ -55,11 +55,11 @@ def _get_message(self, slot):
 
     def _spawn_process(self, message, slot):
         project = message["_project"]
-        environ = self.app.getComponent(IEnvironment)
+        environment = self.app.getComponent(IEnvironment)
         message.setdefault("settings", {})
-        message["settings"].update(environ.get_settings(message))
+        message["settings"].update(environment.get_settings(message))
 
-        env = environ.get_environment(message, slot)
+        env = environment.get_environment(message, slot)
         args = [sys.executable, "-m", self.runner, "crawl", *get_crawl_args(message)]
 
         process = ScrapyProcessProtocol(project, message["_spider"], message["_job"], env, args)

scrapyd/sqlite.py

@@ -1,7 +1,7 @@
+import datetime
 import json
 import os
 import sqlite3
-from datetime import datetime
 
 
 # The database argument is "jobs" (in SqliteJobStorage), or a project (in SqliteSpiderQueue) from get_spider_queues(),
@@ -19,6 +19,19 @@ def initialize(cls, config, database, table):
     return cls(connection_string, table)
 
 
+# https://docs.python.org/3/library/sqlite3.html#sqlite3-adapter-converter-recipes
+def adapt_datetime(val):
+    return val.strftime("%Y-%m-%d %H:%M:%S.%f")
+
+
+def convert_datetime(val):
+    return datetime.datetime.strptime(val.decode(), "%Y-%m-%d %H:%M:%S.%f")
+
+
+sqlite3.register_adapter(datetime.datetime, adapt_datetime)
+sqlite3.register_converter("datetime", convert_datetime)
+
+
 class SqliteMixin:
     def __init__(self, database, table):
         self.database = database or ":memory:"
@@ -139,8 +152,8 @@ def __iter__(self):
                 project,
                 spider,
                 job,
-                datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S.%f"),
-                datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S.%f"),
+                datetime.datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S.%f"),
+                datetime.datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S.%f"),
             )
             for project, spider, job, start_time, end_time in self.conn.execute(
                 f"SELECT project, spider, job, start_time, end_time FROM {self.table} ORDER BY end_time DESC"