From da6ffde2eabb50433aa48000d1eee459b360930f Mon Sep 17 00:00:00 2001 From: Danny van Kooten Date: Wed, 14 Nov 2018 12:40:01 +0100 Subject: [PATCH] speed up host & pathname migration by using UNION. perform VACUUM for SQLite migrations. --- .../sqlstore/migrations/mysql/16_fill_hostnames_table.sql | 3 +-- .../sqlstore/migrations/mysql/17_fill_pathnames_table.sql | 3 +-- .../sqlstore/migrations/postgres/17_fill_hostnames_table.sql | 3 +-- .../sqlstore/migrations/postgres/18_fill_pathnames_table.sql | 3 +-- pkg/datastore/sqlstore/migrations/sqlite3/15_vacuum.sql | 4 ++++ .../sqlstore/migrations/sqlite3/16_fill_hostnames_table.sql | 5 ++--- .../sqlstore/migrations/sqlite3/17_fill_pathnames_table.sql | 3 +-- pkg/datastore/sqlstore/migrations/sqlite3/25_vacuum.sql | 4 ++++ pkg/datastore/sqlstore/sqlstore.go | 2 +- 9 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 pkg/datastore/sqlstore/migrations/sqlite3/15_vacuum.sql create mode 100644 pkg/datastore/sqlstore/migrations/sqlite3/25_vacuum.sql diff --git a/pkg/datastore/sqlstore/migrations/mysql/16_fill_hostnames_table.sql b/pkg/datastore/sqlstore/migrations/mysql/16_fill_hostnames_table.sql index 201f3e2e..5c7c2d26 100644 --- a/pkg/datastore/sqlstore/migrations/mysql/16_fill_hostnames_table.sql +++ b/pkg/datastore/sqlstore/migrations/mysql/16_fill_hostnames_table.sql @@ -1,6 +1,5 @@ -- +migrate Up -INSERT IGNORE INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_page_stats; -INSERT IGNORE INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_referrer_stats; +INSERT INTO hostnames(name) SELECT hostname FROM daily_page_stats UNION SELECT hostname FROM daily_referrer_stats; -- +migrate Down diff --git a/pkg/datastore/sqlstore/migrations/mysql/17_fill_pathnames_table.sql b/pkg/datastore/sqlstore/migrations/mysql/17_fill_pathnames_table.sql index 903758d7..efd39f05 100644 --- a/pkg/datastore/sqlstore/migrations/mysql/17_fill_pathnames_table.sql +++ b/pkg/datastore/sqlstore/migrations/mysql/17_fill_pathnames_table.sql @@ -1,6 +1,5 @@ -- +migrate Up -INSERT IGNORE INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_page_stats; -INSERT IGNORE INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_referrer_stats; +INSERT INTO pathnames(name) SELECT pathname FROM daily_page_stats UNION SELECT pathname FROM daily_referrer_stats; -- +migrate Down diff --git a/pkg/datastore/sqlstore/migrations/postgres/17_fill_hostnames_table.sql b/pkg/datastore/sqlstore/migrations/postgres/17_fill_hostnames_table.sql index 0421e5b4..5c7c2d26 100644 --- a/pkg/datastore/sqlstore/migrations/postgres/17_fill_hostnames_table.sql +++ b/pkg/datastore/sqlstore/migrations/postgres/17_fill_hostnames_table.sql @@ -1,6 +1,5 @@ -- +migrate Up -INSERT INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_page_stats; -INSERT INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_referrer_stats ON CONFLICT(name) DO NOTHING; +INSERT INTO hostnames(name) SELECT hostname FROM daily_page_stats UNION SELECT hostname FROM daily_referrer_stats; -- +migrate Down diff --git a/pkg/datastore/sqlstore/migrations/postgres/18_fill_pathnames_table.sql b/pkg/datastore/sqlstore/migrations/postgres/18_fill_pathnames_table.sql index 52a8adf3..efd39f05 100644 --- a/pkg/datastore/sqlstore/migrations/postgres/18_fill_pathnames_table.sql +++ b/pkg/datastore/sqlstore/migrations/postgres/18_fill_pathnames_table.sql @@ -1,6 +1,5 @@ -- +migrate Up -INSERT INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_page_stats ON CONFLICT(name) DO NOTHING; ; -INSERT INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_referrer_stats ON CONFLICT(name) DO NOTHING; ; +INSERT INTO pathnames(name) SELECT pathname FROM daily_page_stats UNION SELECT pathname FROM daily_referrer_stats; -- +migrate Down diff --git a/pkg/datastore/sqlstore/migrations/sqlite3/15_vacuum.sql b/pkg/datastore/sqlstore/migrations/sqlite3/15_vacuum.sql new file mode 100644 index 00000000..2447e68f --- /dev/null +++ b/pkg/datastore/sqlstore/migrations/sqlite3/15_vacuum.sql @@ -0,0 +1,4 @@ +-- +migrate Up notransaction +VACUUM; + +-- +migrate Down \ No newline at end of file diff --git a/pkg/datastore/sqlstore/migrations/sqlite3/16_fill_hostnames_table.sql b/pkg/datastore/sqlstore/migrations/sqlite3/16_fill_hostnames_table.sql index 502b04c1..248c0d5e 100644 --- a/pkg/datastore/sqlstore/migrations/sqlite3/16_fill_hostnames_table.sql +++ b/pkg/datastore/sqlstore/migrations/sqlite3/16_fill_hostnames_table.sql @@ -1,6 +1,5 @@ --- +migrate Up -INSERT OR IGNORE INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_page_stats; -INSERT OR IGNORE INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_referrer_stats; +-- +migrate Up +INSERT INTO hostnames(name) SELECT hostname FROM daily_page_stats UNION SELECT hostname FROM daily_referrer_stats; -- +migrate Down diff --git a/pkg/datastore/sqlstore/migrations/sqlite3/17_fill_pathnames_table.sql b/pkg/datastore/sqlstore/migrations/sqlite3/17_fill_pathnames_table.sql index 2f868526..efd39f05 100644 --- a/pkg/datastore/sqlstore/migrations/sqlite3/17_fill_pathnames_table.sql +++ b/pkg/datastore/sqlstore/migrations/sqlite3/17_fill_pathnames_table.sql @@ -1,6 +1,5 @@ -- +migrate Up -INSERT OR IGNORE INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_page_stats; -INSERT OR IGNORE INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_referrer_stats; +INSERT INTO pathnames(name) SELECT pathname FROM daily_page_stats UNION SELECT pathname FROM daily_referrer_stats; -- +migrate Down diff --git a/pkg/datastore/sqlstore/migrations/sqlite3/25_vacuum.sql b/pkg/datastore/sqlstore/migrations/sqlite3/25_vacuum.sql new file mode 100644 index 00000000..2447e68f --- /dev/null +++ b/pkg/datastore/sqlstore/migrations/sqlite3/25_vacuum.sql @@ -0,0 +1,4 @@ +-- +migrate Up notransaction +VACUUM; + +-- +migrate Down \ No newline at end of file diff --git a/pkg/datastore/sqlstore/sqlstore.go b/pkg/datastore/sqlstore/sqlstore.go index 7586d28c..902d6d6e 100644 --- a/pkg/datastore/sqlstore/sqlstore.go +++ b/pkg/datastore/sqlstore/sqlstore.go @@ -48,7 +48,7 @@ func New(c *Config) *sqlstore { log.Printf("Connected to %s database: %s on %s", c.Driver, c.Dbname(), c.Host) } - // run migrations + // apply database migrations (if any) db.Migrate() return db