Look ma, multiple DB support (Part I of the DB saga)

.travis.yml

@@ -5,7 +5,7 @@ notifications:
   email: false
 
 before_install:
-  sudo apt-get install -y libsqlite3-dev cppcheck valgrind gcc-4.8
+  sudo apt-get install -y libsqlite3-dev cppcheck valgrind gcc-4.8 gettext
 
 env:
   - ARCH=64 SOURCE_CHECK_ONLY=true COPTFLAGS="-O3"

.travis/build.sh

@@ -29,8 +29,7 @@ if [ ! -f dependencies/bin/bitcoind ]; then
 fi
 
 pyenv global 3.7.1
-pip3 install --user --quiet mako
-pip3 install --user --quiet -r tests/requirements.txt -r doc/requirements.txt
+pip3 install --user --quiet -r requirements.txt -r tests/requirements.txt -r doc/requirements.txt
 pip3 install --quiet \
      pytest-test-groups==1.0.3
 

configure

@@ -259,6 +259,20 @@ int main(void)
 	return 0;
 }
 /*END*/
+var=HAVE_SQLITE3
+desc=sqlite3
+style=DEFINES_EVERYTHING|EXECUTE|MAY_NOT_COMPILE
+link=-lsqlite3
+code=
+#include <sqlite3.h>
+#include <stdio.h>
+
+int main(void)
+{
+	printf("%p\n", sqlite3_prepare_v2);
+	return 0;
+}
+/*END*/
 var=HAVE_GCC
 desc=compiler is GCC
 style=OUTSIDE_MAIN

devtools/sql-rewrite.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+from mako.template import Template
+
+import sys
+
+
+class Sqlite3Rewriter(object):
+    def rewrite(self, query):
+        return query
+
+
+rewriters = {
+    "sqlite3": Sqlite3Rewriter(),
+}
+
+template = Template("""#ifndef LIGHTNINGD_WALLET_GEN_DB_${f.upper()}
+#define LIGHTNINGD_WALLET_GEN_DB_${f.upper()}
+
+#include <config.h>
+#include <wallet/db_common.h>
+
+#if HAVE_${f.upper()}
+
+struct db_query db_${f}_queries[] = {
+
+% for elem in queries:
+    {
+         .name = "${elem['name']}",
+         .query = "${elem['query']}",
+         .placeholders = ${elem['placeholders']},
+         .readonly = ${elem['readonly']},
+    },
+% endfor
+};
+
+#define DB_${f.upper()}_QUERY_COUNT ${len(queries)}
+
+#endif /* HAVE_${f.upper()} */
+
+#endif /* LIGHTNINGD_WALLET_GEN_DB_${f.upper()} */
+""")
+
+
+def extract_queries(pofile):
+    # Given a po-file, extract all queries and their associated names, and
+    # return them as a list.
+
+    def chunk(pofile):
+        # Chunk a given file into chunks separated by an empty line
+        with open(pofile, 'r') as f:
+            chunk = []
+            for line in f:
+                line = line.strip()
+                if line.strip() == "":
+                    yield chunk
+                    chunk = []
+                else:
+                    chunk.append(line.strip())
+            if chunk != []:
+                yield chunk
+
+    queries = []
+    for c in chunk(pofile):
+        name = c[0][3:]
+
+        # Skip other comments
+        i = 1
+        while c[i][0] == '#':
+            i += 1
+
+        # Strip header and surrounding quotes
+        query = c[i][7:][:-1]
+
+        queries.append({
+            'name': name,
+            'query': query,
+            'placeholders': query.count('?'),
+            'readonly': "true" if query.upper().startswith("SELECT") else "false",
+        })
+    return queries
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage:\n\t{} <statements.po-file> <output-dialect>".format(sys.argv[0]))
+        sys.exit(1)
+
+    dialect = sys.argv[2]
+
+    if dialect not in rewriters:
+        print("Unknown dialect {}. The following are available: {}".format(
+            dialect,
+            ", ".join(rewriters.keys())
+        ))
+        sys.exit(1)
+
+    rewriter = rewriters[dialect]
+
+    queries = extract_queries(sys.argv[1])
+    queries = rewriter.rewrite(queries)
+
+    print(template.render(f=dialect, queries=queries))

lightningd/lightningd.c

@@ -572,11 +572,7 @@ static void pidfile_create(const struct lightningd *ld)
  * extra sanity checks, and it's also a good point to free the tmpctx. */
 static int io_poll_lightningd(struct pollfd *fds, nfds_t nfds, int timeout)
 {
-	/*~ In particular, we should *not* have left a database transaction
-	 * open! */
-	db_assert_no_outstanding_statements();
-
-	/* The other checks and freeing tmpctx are common to all daemons. */
+	/* These checks and freeing tmpctx are common to all daemons. */
 	return daemon_poll(fds, nfds, timeout);
 }
 

lightningd/subd.c

@@ -555,7 +555,7 @@ static void destroy_subd(struct subd *sd)
 		sd->channel = NULL;
 
 		/* We can be freed both inside msg handling, or spontaneously. */
-		outer_transaction = db->in_transaction;
+		outer_transaction = db_in_transaction(db);
 		if (!outer_transaction)
 			db_begin_transaction(db);
 		if (sd->errcb)

lightningd/test/run-find_my_abspath.c

@@ -45,9 +45,6 @@ void daemon_setup(const char *argv0 UNNEEDED,
 /* Generated stub for daemon_shutdown */
 void daemon_shutdown(void)
 { fprintf(stderr, "daemon_shutdown called!\n"); abort(); }
-/* Generated stub for db_assert_no_outstanding_statements */
-void db_assert_no_outstanding_statements(void)
-{ fprintf(stderr, "db_assert_no_outstanding_statements called!\n"); abort(); }
 /* Generated stub for db_begin_transaction_ */
 void db_begin_transaction_(struct db *db UNNEEDED, const char *location UNNEEDED)
 { fprintf(stderr, "db_begin_transaction_ called!\n"); abort(); }
@@ -57,6 +54,9 @@ void db_commit_transaction(struct db *db UNNEEDED)
 /* Generated stub for db_get_intvar */
 s64 db_get_intvar(struct db *db UNNEEDED, char *varname UNNEEDED, s64 defval UNNEEDED)
 { fprintf(stderr, "db_get_intvar called!\n"); abort(); }
+/* Generated stub for db_in_transaction */
+bool db_in_transaction(struct db *db UNNEEDED)
+{ fprintf(stderr, "db_in_transaction called!\n"); abort(); }
 /* Generated stub for fatal */
 void   fatal(const char *fmt UNNEEDED, ...)
 { fprintf(stderr, "fatal called!\n"); abort(); }

requirements.txt

@@ -0,0 +1,3 @@
+sqlparse==0.3.0
+mako==1.0.14
+mrkd==0.1.5

tests/plugins/dblog.py

@@ -16,9 +16,18 @@ def init(configuration, options, plugin):
     plugin.conn = sqlite3.connect(plugin.get_option('dblog-file'),
                                   isolation_level=None)
     plugin.log("replaying pre-init data:")
+    plugin.conn.execute("PRAGMA foreign_keys = ON;")
+
+    print(plugin.sqlite_pre_init_cmds)
+
+    plugin.conn.execute("BEGIN TRANSACTION;")
+
     for c in plugin.sqlite_pre_init_cmds:
         plugin.conn.execute(c)
         plugin.log("{}".format(c))
+
+    plugin.conn.execute("COMMIT;")
+
     plugin.initted = True
     plugin.log("initialized {}".format(configuration))
 
@@ -29,9 +38,15 @@ def db_write(plugin, writes, **kwargs):
         plugin.log("deferring {} commands".format(len(writes)))
         plugin.sqlite_pre_init_cmds += writes
     else:
+        print(writes)
+        plugin.conn.execute("BEGIN TRANSACTION;")
+
         for c in writes:
             plugin.conn.execute(c)
             plugin.log("{}".format(c))
+
+        plugin.conn.execute("COMMIT;")
+
     return True
 
 

tests/test_plugin.py

@@ -299,10 +299,9 @@ def test_db_hook(node_factory, executor):
     # It should see the db being created, and sometime later actually get
     # initted.
     # This precedes startup, so needle already past
-    assert l1.daemon.is_in_log('plugin-dblog.py deferring 1 commands')
+    assert l1.daemon.is_in_log(r'plugin-dblog.py deferring \d+ commands')
     l1.daemon.logsearch_start = 0
     l1.daemon.wait_for_log('plugin-dblog.py replaying pre-init data:')
-    l1.daemon.wait_for_log('plugin-dblog.py PRAGMA foreign_keys = ON;')
     l1.daemon.wait_for_log('plugin-dblog.py CREATE TABLE version \\(version INTEGER\\)')
     l1.daemon.wait_for_log("plugin-dblog.py initialized.* 'startup': True")
 

wallet/Makefile

@@ -11,7 +11,10 @@ WALLET_LIB_SRC :=		\
 	wallet/wallet.c		\
 	wallet/walletrpc.c
 
-WALLET_LIB_OBJS := $(WALLET_LIB_SRC:.c=.o)
+WALLET_DB_DRIVERS :=		\
+	wallet/db_sqlite3.c
+
+WALLET_LIB_OBJS := $(WALLET_LIB_SRC:.c=.o) $(WALLET_DB_DRIVERS:.c=.o)
 WALLET_LIB_HEADERS := $(WALLET_LIB_SRC:.c=.h)
 
 # Make sure these depend on everything.
@@ -26,7 +29,25 @@ check-source-bolt: $(WALLET_LIB_SRC:%=bolt-check/%) $(WALLET_LIB_HEADERS:%=bolt-
 
 clean: wallet-clean
 
+wallet/db_sqlite3.c: wallet/gen_db_sqlite3.c
+
+# The following files contain SQL-annotated statements that we need to extact
+SQL_FILES := 				\
+	wallet/db.c 			\
+	wallet/invoices.c		\
+	wallet/wallet.c 		\
+	wallet/test/run-db.c		\
+	wallet/test/run-wallet.c	\
+
+wallet/statements.po: $(SQL_FILES)
+	xgettext -kNAMED_SQL -kSQL --add-location --no-wrap --omit-header -o $@ $(SQL_FILES)
+
+wallet/gen_db_sqlite3.c:  wallet/statements.po devtools/sql-rewrite.py
+	 devtools/sql-rewrite.py wallet/statements.po sqlite3 > wallet/gen_db_sqlite3.c
+
 wallet-clean:
 	$(RM) $(WALLET_LIB_OBJS)
+	$(RM) wallet/statements.po
+	$(RM) wallet/gen_db_sqlite3.c
 
 include wallet/test/Makefile