speed up host & pathname migration by using UNION. perform VACUUM for…

… SQLite migrations.

pkg/datastore/sqlstore/migrations/mysql/16_fill_hostnames_table.sql

@@ -1,6 +1,5 @@
 -- +migrate Up
-INSERT IGNORE INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_page_stats;
-INSERT IGNORE INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_referrer_stats; 
+INSERT INTO hostnames(name) SELECT hostname FROM daily_page_stats UNION SELECT hostname FROM daily_referrer_stats;
 
 -- +migrate Down
 

pkg/datastore/sqlstore/migrations/mysql/17_fill_pathnames_table.sql

@@ -1,6 +1,5 @@
 -- +migrate Up
-INSERT IGNORE INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_page_stats;
-INSERT IGNORE INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_referrer_stats;
+INSERT INTO pathnames(name) SELECT pathname FROM daily_page_stats UNION SELECT pathname FROM daily_referrer_stats;
 
 -- +migrate Down
 

pkg/datastore/sqlstore/migrations/postgres/17_fill_hostnames_table.sql

@@ -1,6 +1,5 @@
 -- +migrate Up
-INSERT INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_page_stats;
-INSERT INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_referrer_stats ON CONFLICT(name) DO NOTHING; 
+INSERT INTO hostnames(name) SELECT hostname FROM daily_page_stats UNION SELECT hostname FROM daily_referrer_stats;
 
 -- +migrate Down
 

pkg/datastore/sqlstore/migrations/postgres/18_fill_pathnames_table.sql

@@ -1,6 +1,5 @@
 -- +migrate Up
-INSERT INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_page_stats ON CONFLICT(name) DO NOTHING; ;
-INSERT INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_referrer_stats ON CONFLICT(name) DO NOTHING; ;
+INSERT INTO pathnames(name) SELECT pathname FROM daily_page_stats UNION SELECT pathname FROM daily_referrer_stats;
 
 -- +migrate Down
 

pkg/datastore/sqlstore/migrations/sqlite3/15_vacuum.sql

@@ -0,0 +1,4 @@
+-- +migrate Up  notransaction
+VACUUM;
+
+-- +migrate Down 

pkg/datastore/sqlstore/migrations/sqlite3/16_fill_hostnames_table.sql

@@ -1,6 +1,5 @@
--- +migrate Up
-INSERT OR IGNORE INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_page_stats;
-INSERT OR IGNORE INTO hostnames(name) SELECT DISTINCT(hostname) FROM daily_referrer_stats; 
+-- +migrate Up 
+INSERT INTO hostnames(name) SELECT hostname FROM daily_page_stats UNION SELECT hostname FROM daily_referrer_stats;
 
 -- +migrate Down
 

pkg/datastore/sqlstore/migrations/sqlite3/17_fill_pathnames_table.sql

@@ -1,6 +1,5 @@
 -- +migrate Up
-INSERT OR IGNORE INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_page_stats;
-INSERT OR IGNORE INTO pathnames(name) SELECT DISTINCT(pathname) FROM daily_referrer_stats;
+INSERT INTO pathnames(name) SELECT pathname FROM daily_page_stats UNION SELECT pathname FROM daily_referrer_stats;
 
 -- +migrate Down
 

pkg/datastore/sqlstore/migrations/sqlite3/25_vacuum.sql

@@ -0,0 +1,4 @@
+-- +migrate Up  notransaction
+VACUUM;
+
+-- +migrate Down 

pkg/datastore/sqlstore/sqlstore.go

@@ -48,7 +48,7 @@ func New(c *Config) *sqlstore {
 		log.Printf("Connected to %s database: %s on %s", c.Driver, c.Dbname(), c.Host)
 	}
 
-	// run migrations
+	// apply database migrations (if any)
 	db.Migrate()
 
 	return db