[MOS-855] Restore process update

Fixed issues with restoring from legacy backups
and cases when the database set fetched from backup package
is different than the system one.

Author: Mateusz Piesta

.gitignore

@@ -68,10 +68,16 @@ test/test_env
 image/luts.bin
 image/assets/fonts/*
 
+scripts/lua/migration/test/databases/*
+scripts/lua/migration/test/migrations/*
 scripts/lua/test/assets*
-scripts/lua/test/update/user/temp/*
+scripts/lua/test/device/user/temp/recovery_status.json
+scripts/lua/test/update/user/*
 scripts/lua/test/update/target/*
 scripts/lua/test/update/system/*
-scripts/lua/test/update_udm/user/temp/*
+scripts/lua/test/update_udm/user/*
 scripts/lua/test/update_udm/target/*
 scripts/lua/test/update_udm/system/*
+scripts/lua/test/restore/user/*
+scripts/lua/test/restore/target/*
+scripts/lua/test/restore/system/*

scripts/lua/entry.lua

@@ -37,7 +37,7 @@ local function generate_report_file(boot_reason_str, success, message)
     local body = string.format(
         "{\"version\": \"%s\",\"branch\": \"%s\",\"revision\": \"%s\",\"operation\": \"%s\",\"successful\": %s,\"message\": \"%s\"}",
         rec.version(), rec.branch(), rec.revision(), boot_reason_str, tostring(success), message)
-    local fd = io.open(file_path, 'w')
+    local fd = assert(io.open(file_path, 'w'))
     fd:write(body)
     fd:close()
 end

scripts/lua/migration/migration.lua

@@ -37,10 +37,10 @@ local function build_database_path(path, db_name)
 end
 
 local function db_exec(file, script, version)
-    local db = assert(sqlite3.open(file), string.format("file: %s", file))
-    assert(db:exec(script) == sqlite3.OK, string.format("script:\n%s\n", script))
+    local db = assert(sqlite3.open(file))
+    assert(db:exec(script) == sqlite3.OK, string.format("Script execution failed:\n%s\n", script))
     assert(db:exec(string.format("PRAGMA user_version=%u;", version)) == sqlite3.OK,
-        string.format("version: %d", version))
+        string.format("Setting database version: %d failed", version))
     db:close()
 end
 
@@ -90,26 +90,33 @@ local function db_migrate_down(db_path, scripts, target_version)
 end
 
 local function print_db_set(db_set)
-    print("database set:")
+    print("Database set(name,version):")
     for name, version in pairs(db_set) do
-        print(string.format("'%s':%d",name,version))
+        print(string.format("   '%s':%d", name, version))
     end
 end
 
 local function validate_inputs(migration_dir, db_dir)
-    assert(helpers.exists(migration_dir), "Migrations directory does not exist")
+    assert(helpers.exists(migration_dir), "Migration directory does not exist")
     assert(helpers.exists(db_dir), "Databases directory does not exist")
     return true
 end
 
 local function migrate(db_path, scripts_up, scripts_down, target_version)
     local db_version = read_db_version(db_path)
+
     if db_version > target_version then
+        print(string.format("Performing migration-down of '%s' from version %d to %d", db_path, db_version,
+            target_version))
         return migration.down(db_path, scripts_down, target_version)
     end
     if db_version < target_version then
+        print(
+            string.format("Performing migration-up of '%s' from version %d to %d", db_path, db_version, target_version))
         return migration.up(db_path, scripts_up, target_version)
     end
+
+    print(string.format("Migration not needed, '%s' is already the newest version", db_path))
     return migration.retcode.OK
 end
 
@@ -148,8 +155,8 @@ end
 -- @param db_set array of {<"database_name"> = <db_target_version>} entries
 -- @return @{retcode}
 function migration.migrate(db_dir, scripts_dir, db_set)
-    print(string.format("migrations scripts directory: '%s'", scripts_dir))
-    print(string.format("databases directory: '%s'", db_dir))
+    print(string.format("Migration scripts directory: '%s'", scripts_dir))
+    print(string.format("Databases directory: '%s'", db_dir))
     print_db_set(db_set)
 
     for name, version in pairs(db_set) do

scripts/lua/migration/test/test.lua

@@ -1,6 +1,5 @@
---Copyright (c) 2017-2022, Mudita Sp. z.o.o. All rights reserved.
---For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md
-
+-- Copyright (c) 2017-2022, Mudita Sp. z.o.o. All rights reserved.
+-- For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md
 package.path = package.path .. ";../?.lua;../../share/?.lua;../../share/?/?.lua"
 local lu = require("luaunit")
 local sqlite = require("lsqlite3complete")
@@ -41,35 +40,35 @@ function test_migration_up_success()
     scripts[5] = "ALTER TABLE test_1 ADD new_column5 TEXT;"
 
     --- Migrate from version 0 to 1
-    spawn_db("test_1.sql", 0, test_db_schema)
-    lu.assertEquals(migration.up("test_1.sql", scripts, 1), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 1)
+    spawn_db("databases/test_1.sql", 0, test_db_schema)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 1), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 1)
 
     -- Migrate from version 1 to 2
-    lu.assertEquals(migration.up("test_1.sql", scripts, 2), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 2)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 2), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 2)
 
     --- Migrate from version 0 to 5
-    spawn_db("test_1.sql", 0, test_db_schema)
-    lu.assertEquals(migration.up("test_1.sql", scripts, 5), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 5)
+    spawn_db("databases/test_1.sql", 0, test_db_schema)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 5), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 5)
 end
 
 function test_migration_up_db_errors()
     scripts = {}
 
     --- Trigger DB error by trying to operate on empty database
-    spawn_db("test_1.sql", 1, "")
+    spawn_db("databases/test_1.sql", 1, "")
     scripts[1] = "ALTER TABLE test_1 ADD new_column TEXT;"
-    lu.assertError(migration.up, "test_1.sql", scripts, 2)
+    lu.assertError(migration.up, "databases/test_1.sql", scripts, 2)
 
     --- Target version set to the same value as the current DB version
-    spawn_db("test_1.sql", 1, test_db_schema)
-    lu.assertEquals(migration.up("test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE)
+    spawn_db("databases/test_1.sql", 1, test_db_schema)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE)
 
     --- Target version set to the lower number than the current DB version
-    spawn_db("test_1.sql", 2, test_db_schema)
-    lu.assertEquals(migration.up("test_1.sql", scripts, 1), migration.retcode.WRONG_TARGET_VERSION)
+    spawn_db("databases/test_1.sql", 2, test_db_schema)
+    lu.assertEquals(migration.up("databases/test_1.sql", scripts, 1), migration.retcode.WRONG_TARGET_VERSION)
 end
 
 function test_migration_down_success()
@@ -81,35 +80,35 @@ function test_migration_down_success()
     scripts[5] = "ALTER TABLE test_1 ADD new_column5 TEXT;"
 
     --- Migrate from version 2 to 1
-    spawn_db("test_1.sql", 2, test_db_schema)
-    lu.assertEquals(migration.down("test_1.sql", scripts, 1), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 1)
+    spawn_db("databases/test_1.sql", 2, test_db_schema)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 1), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 1)
 
     -- Migrate from version 1 to 0
-    lu.assertEquals(migration.down("test_1.sql", scripts, 0), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 0)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 0), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 0)
 
     --- Migrate from version 5 to 0
-    spawn_db("test_1.sql", 5, test_db_schema)
-    lu.assertEquals(migration.down("test_1.sql", scripts, 0), migration.retcode.OK)
-    lu.assertEquals(migration.get_db_version("test_1.sql"), 0)
+    spawn_db("databases/test_1.sql", 5, test_db_schema)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 0), migration.retcode.OK)
+    lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 0)
 end
 
 function test_migration_down_errors()
     scripts = {}
 
     --- Trigger DB error by trying to operate on empty database
-    spawn_db("test_1.sql", 2, "")
+    spawn_db("databases/test_1.sql", 2, "")
     scripts[1] = "ALTER TABLE test_1 ADD new_column TEXT;"
-    lu.assertError(migration.down, "test_1.sql", scripts, 1)
+    lu.assertError(migration.down, "databases/test_1.sql", scripts, 1)
 
     --- Target version set to the same value as the current DB version
-    spawn_db("test_1.sql", 1, test_db_schema)
-    lu.assertEquals(migration.down("test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE)
+    spawn_db("databases/test_1.sql", 1, test_db_schema)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE)
 
     --- Target version set to the higher number than the current DB version
-    spawn_db("test_1.sql", 2, test_db_schema)
-    lu.assertEquals(migration.down("test_1.sql", scripts, 3), migration.retcode.WRONG_TARGET_VERSION)
+    spawn_db("databases/test_1.sql", 2, test_db_schema)
+    lu.assertEquals(migration.down("databases/test_1.sql", scripts, 3), migration.retcode.WRONG_TARGET_VERSION)
 end
 
 function test_automatic_migration()
@@ -124,6 +123,13 @@ function test_automatic_migration()
     spawn_script("down.sql", "migrations", "test_2", 1, "delete from test_1 where _id=1;")
     spawn_script("down.sql", "migrations", "test_2", 2, "delete from test_1 where _id=2;")
 
+    migration.migrate("databases", "migrations", {
+        test_1 = 0,
+        test_2 = 0
+    })
+    lu.assertEquals(migration.get_db_version("databases/test_1.db"), 0)
+    lu.assertEquals(migration.get_db_version("databases/test_2.db"), 0)
+
     migration.migrate("databases", "migrations", {
         test_1 = 2,
         test_2 = 2

scripts/lua/restore.lua

@@ -9,8 +9,6 @@ local migration = require('migration')
 local restore = {}
 
 local unpacked_backup_dir = paths.temp_dir .. "/backup"
-local version_file = unpacked_backup_dir .. "/" .. consts.version_file
-local legacy_version_file = unpacked_backup_dir .. "/" .. consts.legacy_version_file
 
 restore.script_name = "restore.lua"
 restore.img_in_progress = "assets/gui_image_restore_in_progress.bin"
@@ -37,7 +35,7 @@ local function unpack_backup()
     ltar.unpack(paths.backup_file, unpacked_backup_dir)
 end
 
-local function build_db_set(file)
+local function get_db_array_from_file(file)
     local contents = helpers.read_whole_file(file)
     local root = json.decode(contents)
     local set = {}
@@ -47,50 +45,48 @@ local function build_db_set(file)
     return set
 end
 
-local function get_legacy_db_set()
-    local set = {
-        ["calllog"] = 0,
-        ["sms"] = 0,
-        ["events"] = 0,
-        ["settings_v2"] = 0,
-        ["notes"] = 0,
-        ["custom_quotes"] = 0,
-        ["predefined_quotes"] = 0,
-        ["contacts"] = 0,
-        ["alarms"] = 0,
-        ["notifications"] = 0,
-        ["multimedia"] = 0
-    }
+local function get_db_array_from_path(path)
+    local set = {}
+    for file in lfs.dir(path) do
+        local file_path = path .. "/" .. file
+        if file ~= "." and file ~= ".." then
+            if lfs.attributes(file_path, "mode") == "file" then
+                set[helpers.strip_from_extension(file)] = true;
+            end
+        end
+    end
+    return set
+end
+
+local function build_db_set()
+    local system_db_set = get_db_array_from_file(paths.version_file)
+    local backup_db_set = get_db_array_from_path(unpacked_backup_dir)
+    local set = {}
+    for name, version in pairs(system_db_set) do
+        if backup_db_set[name] then
+            set[name] = tonumber(version)
+        end
+    end
     return set
 end
 
 local function perform_db_migration()
     print("Performing database migration")
-    local dbset = {}
-    if helpers.exists(version_file) then
-        dbset = build_db_set(version_file)
-    else
-        assert(helpers.exists(legacy_version_file))
-        print("Legacy backup file, assuming legacy databases set")
-        dbset = get_legacy_db_set()
-    end
 
-    local result = migration.migrate(unpacked_backup_dir, paths.migration_scripts_dir, dbset)
+    local result = migration.migrate(unpacked_backup_dir, paths.migration_scripts_dir, build_db_set())
     assert(result == migration.retcode.OK, string.format("Database migration process failed with %d", result))
 end
 
 local function sync_databases()
     print("Syncing databases:")
-    print(string.format("Replacing old databases: '%s' with the ones from '%s'", paths.db_dir, unpacked_backup_dir))
-
-    helpers.rm_files_from_dir(paths.db_dir)
-    helpers.copy_dir(unpacked_backup_dir, paths.db_dir)
 
-    local version_file_path = paths.db_dir .. "/" .. consts.version_file
-    if not helpers.exists(version_file_path) then
-        version_file_path = paths.db_dir .. "/" .. consts.legacy_version_file
+    for name, _ in pairs(build_db_set()) do
+        local destination = paths.db_dir .. "/" .. name .. ".db"
+        local source = unpacked_backup_dir .. "/" .. name .. ".db"
+        print(string.format("Replacing '%s' with '%s'", destination, source));
+        assert(os.remove(destination))
+        helpers.copy_file(source, destination)
     end
-    assert(os.remove(version_file_path))
 end
 
 local function remove_cache()

scripts/lua/share/consts.lua

@@ -1,7 +1,12 @@
 local consts = {}
 
 consts.version_file = "version.json"
-consts.legacy_version_file = "backup.json" -- Pre-UDM backup package had version.json file named as backup.json
 consts.indexer_cache_file = ".directory_is_indexed"
 
+local match = {}
+
+-- Match only files with '.db' extensions and omit such files inside subdirectories
+match.only_db_files = '^[^%/]*%.db$'
+
+consts.match = match
 return consts

scripts/lua/share/helpers.lua

@@ -152,7 +152,7 @@ function helpers.move_dir(from, where)
         if attr.mode == "directory" then
             assert(lfs.mkdir(build_path(where, name)))
         else
-            assert(os.rename(build_path(from, name),build_path(where, name)))
+            assert(os.rename(build_path(from, name), build_path(where, name)))
         end
     end
 end
@@ -235,6 +235,14 @@ function helpers.get_file_extension(file)
     return file:match("^.+(%..+)$")
 end
 
+--- Strips file name from its extension
+-- @function strip_from_extension
+-- @param file file path
+-- @return file name without extension
+function helpers.strip_from_extension(file)
+    return file:match("^(.*)%..*$")
+end
+
 --- Create directory and all required subdirectories
 -- @function mkdirp
 -- @param file file path