add sqlite support for mega

common/src/config.rs

@@ -48,6 +48,8 @@ impl Default for LogConfig {
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct DbConfig {
+    pub db_type: String,
+    pub db_path: String,
     pub db_url: String,
     pub max_connection: u32,
     pub min_connection: u32,
@@ -57,6 +59,8 @@ pub struct DbConfig {
 impl Default for DbConfig {
     fn default() -> Self {
         Self {
+            db_type: String::new(),
+            db_path: String::new(),
             db_url: String::new(),
             max_connection: 32,
             min_connection: 16,

docs/development.md

@@ -23,7 +23,7 @@
    $ cargo build
    ```
 
-3. Install PostgreSQL and init database.
+3. Install PostgreSQL and init database. (You can skip this step if using SQLite in `config.toml`)
 
    1. Install PostgreSQL 16 with `brew` command.
 
@@ -90,8 +90,15 @@
 
 
       [database]
+      # "sqlite" | "postgres"
+      # "sqlite" will use `db_path` and ignore `db_url`
+      db_type = "sqlite"
+
+      # used for sqlite
+      db_path = "/tmp/.mega/mega.db"
+
       # database connection url
-      db_url = "postgres://postgres:postgres@localhost:5432/mega"
+      db_url = "postgres://mega:mega@localhost:5432/mega"
 
       # db max connection, setting it to twice the number of CPU cores would be appropriate.
       max_connection = 32
@@ -188,7 +195,7 @@
    $ cargo build
    ```
 
-3. Install PostgreSQL and initialize database.
+3. Install PostgreSQL and initialize database. (You can skip this step if using SQLite in `config.toml`)
 
    1.Install PostgreSQL.
 
@@ -261,6 +268,13 @@
 
 
       [database]
+      # "sqlite" | "postgres"
+      # "sqlite" will use `db_path` and ignore `db_url`
+      db_type = "sqlite"
+
+      # used for sqlite
+      db_path = "/tmp/.mega/mega.db"
+
       # database connection url
       db_url = "postgres://mega:mega@localhost:5432/mega"
 
@@ -356,6 +370,8 @@
 
 If you are using GitHub codespaces, you can follow the steps below to set up the Mega project. When you create a new Codespace, the Mega project will be cloned automatically. You can then follow the steps below to set up the project.
 
+You can skip this step (PostgreSQL setup) if using SQLite in `config.toml`.
+
 When the codespace is ready, the PostgreSQL will be installed and started automatically. You can then follow the steps below to set up the database with below steps.
 
 ```bash
@@ -390,6 +406,13 @@ Config `confg.toml` file for the Mega project.
 
 
    [database]
+   # "sqlite" | "postgres"
+   # "sqlite" will use `db_path` and ignore `db_url`
+   db_type = "sqlite"
+
+   # used for sqlite
+   db_path = "/tmp/.mega/mega.db"
+
    # database connection url
    db_url = "postgres://mega:mega@localhost:5432/mega"
 
@@ -448,6 +471,14 @@ Config `confg.toml` file for the Mega project.
    # Size of each file chunk when splitting is enabled, in bytes. Ignored if splitting is disabled.
    split_size = 20971520 # Default size is 20MB (20971520 bytes)
 ```
+## Database maintenance
+Currently, the tables of database are created by `.sql` file. 
+
+If you want to add a new table or modify the existing table, you need to update the `.sql` files which are located in the `sql` directory.
+
+### Attention
+- Each database corresponds to one `.sql` file, you must modify all of them if you want to update the tables in order to keep the consistency of the database.
+- DO NOT use `Array` Type in PostgreSQL but use `JSON` instead, for compatibility with SQLite & MySQL. (`JSON` <==> `serde_json::Value`)
 
 ## Comment Guideline
 

jupiter/Cargo.toml

@@ -18,6 +18,7 @@ mercury = { workspace = true }
 sea-orm = { workspace = true, features = [
     "sqlx-postgres",
     "sqlx-mysql",
+    "sqlx-sqlite",
     "runtime-tokio-rustls",
     "macros",
 ] }

jupiter/callisto/Cargo.toml

@@ -12,5 +12,6 @@ path = "src/lib.rs"
 common = { path = "../../common" }
 
 serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
 chrono = { workspace = true }
-sea-orm = { workspace = true, features = ["sqlx-postgres", "sqlx-mysql"] }
+sea-orm = { workspace = true, features = ["sqlx-postgres", "sqlx-mysql", "sqlx-sqlite"] }

jupiter/callisto/src/git_commit.rs

@@ -1,6 +1,7 @@
 //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
 
 use sea_orm::entity::prelude::*;
+use serde_json::Value;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "git_commit")]
@@ -10,7 +11,7 @@ pub struct Model {
     pub repo_id: i64,
     pub commit_id: String,
     pub tree: String,
-    pub parents_id: Vec<String>,
+    pub parents_id: Value,
     #[sea_orm(column_type = "Text", nullable)]
     pub author: Option<String>,
     #[sea_orm(column_type = "Text", nullable)]

jupiter/callisto/src/mega_commit.rs

@@ -1,6 +1,7 @@
 //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
 
 use sea_orm::entity::prelude::*;
+use serde_json::Value;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "mega_commit")]
@@ -10,7 +11,7 @@ pub struct Model {
     #[sea_orm(unique)]
     pub commit_id: String,
     pub tree: String,
-    pub parents_id: Vec<String>,
+    pub parents_id: Value,
     #[sea_orm(column_type = "Text", nullable)]
     pub author: Option<String>,
     #[sea_orm(column_type = "Text", nullable)]

jupiter/src/storage/init.rs

@@ -1,6 +1,6 @@
-use std::time::Duration;
+use std::{path::Path, time::Duration};
 
-use sea_orm::{ConnectOptions, Database, DatabaseConnection};
+use sea_orm::{ConnectOptions, ConnectionTrait, Database, DatabaseConnection, DbErr, Statement, TransactionError, TransactionTrait};
 use tracing::log;
 
 use common::config::DbConfig;
@@ -9,7 +9,21 @@ use crate::utils::id_generator;
 
 pub async fn database_connection(db_config: &DbConfig) -> DatabaseConnection {
     id_generator::set_up_options().unwrap();
-    let mut opt = ConnectOptions::new(db_config.db_url.to_owned());
+
+    let is_sqlite = db_config.db_type == "sqlite";
+    let db_path = &db_config.db_path;
+    let db_url = if is_sqlite {
+        if !Path::new(db_path).exists() {
+            log::info!("Creating new sqlite database: {}", db_path);
+            std::fs::File::create(db_path).expect("Failed to create sqlite database");
+        }
+        &format!("sqlite://{}", db_path)
+    } else {
+        &db_config.db_url
+    };
+    log::info!("Connecting to database: {}", db_url);
+
+    let mut opt = ConnectOptions::new(db_url.to_owned());
     opt.max_connections(db_config.max_connection)
         .min_connections(db_config.min_connection)
         .acquire_timeout(Duration::from_secs(30))
@@ -18,7 +32,34 @@ pub async fn database_connection(db_config: &DbConfig) -> DatabaseConnection {
         .max_lifetime(Duration::from_secs(8))
         .sqlx_logging(db_config.sqlx_logging)
         .sqlx_logging_level(log::LevelFilter::Debug);
-    Database::connect(opt)
+    let conn = Database::connect(opt)
         .await
-        .expect("Database connection failed")
+        .expect("Database connection failed");
+
+    // setup sqlite database (execute .sql)
+    if is_sqlite && is_file_empty(db_path) {
+        log::info!("Setting up sqlite database");
+        setup_sql(&conn).await.expect("Failed to setup sqlite database");
+    }
+    conn
 }
+
+/// create table from .sql file
+async fn setup_sql(conn: &DatabaseConnection) -> Result<(), TransactionError<DbErr>> {
+    conn.transaction::<_, _, DbErr>(|txn| {
+        Box::pin(async move {
+            let backend = txn.get_database_backend();
+
+            // `include_str!` will expand the file while compiling, so `.sql` is not needed after that
+            const SETUP_SQL: &str = include_str!("../../../sql/sqlite/sqlite_20240711_init.sql");
+            txn.execute(Statement::from_string(backend, SETUP_SQL)).await?;
+            Ok(())
+        })
+    })
+    .await
+}
+
+fn is_file_empty(path: &str) -> bool {
+    let metadata = std::fs::metadata(path).unwrap();
+    metadata.len() == 0
+}

mega/config.toml

@@ -12,6 +12,13 @@ print_std = true
 
 
 [database]
+# "sqlite" | "postgres"
+# "sqlite" will use `db_path` and ignore `db_url`
+db_type = "sqlite"
+
+# used for sqlite
+db_path = "/tmp/.mega/mega.db"
+
 # database connection url
 db_url = "postgres://mega:mega@localhost:5432/mega"
 

mercury/src/internal/model/commit.rs

@@ -15,8 +15,10 @@ impl From<mega_commit::Model> for Commit {
             tree_id: SHA1::from_str(&value.tree).unwrap(),
             parent_commit_ids: value
                 .parents_id
-                .into_iter()
-                .map(|id| SHA1::from_str(&id).unwrap())
+                .as_array()
+                .unwrap()
+                .iter()
+                .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap())
                 .collect(),
             author: Signature::from_data(value.author.unwrap().into()).unwrap(),
             committer: Signature::from_data(value.committer.unwrap().into()).unwrap(),
@@ -32,8 +34,10 @@ impl From<git_commit::Model> for Commit {
             tree_id: SHA1::from_str(&value.tree).unwrap(),
             parent_commit_ids: value
                 .parents_id
-                .into_iter()
-                .map(|id| SHA1::from_str(&id).unwrap())
+                .as_array()
+                .unwrap()
+                .iter()
+                .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap())
                 .collect(),
             author: Signature::from_data(value.author.unwrap().into()).unwrap(),
             committer: Signature::from_data(value.committer.unwrap().into()).unwrap(),

sql/postgres/pg_20240205__init.sql

@@ -2,7 +2,7 @@ CREATE TABLE IF NOT EXISTS "mega_commit" (
   "id" BIGINT PRIMARY KEY,
   "commit_id" VARCHAR(40) NOT NULL,
   "tree" VARCHAR(40) NOT NULL,
-  "parents_id" TEXT [] NOT NULL,
+  "parents_id" JSON NOT NULL,  -- for compatibility with sqlite, DO NOT use Array Type
   "author" TEXT,
   "committer" TEXT,
   "content" TEXT,
@@ -117,7 +117,7 @@ CREATE TABLE IF NOT EXISTS "git_commit" (
   "repo_id" BIGINT NOT NULL,
   "commit_id" VARCHAR(40) NOT NULL,
   "tree" VARCHAR(40) NOT NULL,
-  "parents_id" TEXT [] NOT NULL,
+  "parents_id" JSON NOT NULL,
   "author" TEXT,
   "committer" TEXT,
   "content" TEXT,