diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index d64e9110..21a11b23 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -163,6 +163,10 @@ jobs: name: Make Build Clean run: cargo clean - - id: test - name: Run Integration Tests - run: ./contrib/dev-tools/container/e2e/run-e2e-tests.sh + - id: test-sqlite + name: Run Integration Tests (SQLite) + run: ./contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh + + - id: test-mysql + name: Run Integration Tests (MySQL) + run: ./contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh diff --git a/Cargo.lock b/Cargo.lock index 8c94b306..90e50a75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3008,6 +3008,7 @@ dependencies = [ "rand_core", "regex", "reqwest", + "rustversion", "serde", "serde_bencode", "serde_bytes", diff --git a/Cargo.toml b/Cargo.toml index 802cadaa..cc884b39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ pbkdf2 = { version = "0", features = ["simple"] } rand_core = { version = "0", features = ["std"] } regex = "1" reqwest = { version = "0", features = ["json", "multipart"] } +rustversion = "1.0.14" serde = { version = "1", features = ["rc"] } serde_bencode = "0" serde_bytes = "0" diff --git a/compose.yaml b/compose.yaml index a7f9e881..3ad56e74 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,7 +5,7 @@ services: build: context: . dockerfile: ./Containerfile - target: debug + target: release tty: true environment: - TORRUST_INDEX_CONFIG=${TORRUST_INDEX_CONFIG} diff --git a/contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh b/contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh index 5124e5dc..73cd2fc6 100755 --- a/contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh +++ b/contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh @@ -4,13 +4,12 @@ TORRUST_INDEX_CONFIG=$(cat ./share/default/config/index.container.mysql.toml) \ docker compose build USER_ID=${USER_ID:-1000} \ - # Index TORRUST_INDEX_CONFIG=$(cat ./share/default/config/index.container.mysql.toml) \ TORRUST_INDEX_DATABASE_DRIVER="mysql" \ TORRUST_INDEX_TRACKER_API_TOKEN="MyAccessToken" \ TORRUST_IDX_BACK_MYSQL_DATABASE="torrust_index_e2e_testing" \ - # Tracker - TORRUST_TRACKER_CONFIG=$(cat ./share/default/config/tracker.container.mysql.toml) \ - TORRUST_TRACKER_DATABASE_DRIVER="mysql" \ + TORRUST_TRACKER_CONFIG=$(cat ./share/default/config/tracker.container.sqlite3.toml) \ + TORRUST_TRACKER_DATABASE_DRIVER="sqlite3" \ TORRUST_TRACKER_API_ADMIN_TOKEN="MyAccessToken" \ docker compose up -d + diff --git a/contrib/dev-tools/container/e2e/mysql/install.sh b/contrib/dev-tools/container/e2e/mysql/install.sh new file mode 100755 index 00000000..de8e1cf3 --- /dev/null +++ b/contrib/dev-tools/container/e2e/mysql/install.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# This script is only intended to be used for E2E testing environment. + +# Database credentials +MYSQL_USER="root" +MYSQL_PASSWORD="root_secret_password" +MYSQL_HOST="127.0.0.1" +MYSQL_DATABASE="torrust_index_e2e_testing" + +# Create the MySQL database for the index. Assumes MySQL client is installed. +# The docker compose configuration already creates the database the first time +# the container is created. +echo "Creating MySQL database $MYSQL_DATABASE for for E2E testing ..." +MYSQL_PWD=$MYSQL_PASSWORD mysql -h $MYSQL_HOST -u $MYSQL_USER -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE;" diff --git a/contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh b/contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh new file mode 100755 index 00000000..a9e275d2 --- /dev/null +++ b/contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +CURRENT_USER_NAME=$(whoami) +CURRENT_USER_ID=$(id -u) +echo "User name: $CURRENT_USER_NAME" +echo "User id: $CURRENT_USER_ID" + +TORRUST_IDX_BACK_USER_UID=$CURRENT_USER_ID +TORRUST_TRACKER_USER_UID=$CURRENT_USER_ID +export TORRUST_IDX_BACK_USER_UID +export TORRUST_TRACKER_USER_UID + +# todo: remove duplicate funtion +wait_for_container_to_be_healthy() { + local container_name="$1" + local max_retries="$2" + local retry_interval="$3" + local retry_count=0 + + while [ $retry_count -lt "$max_retries" ]; do + container_health="$(docker inspect --format='{{json .State.Health}}' "$container_name")" + if [ "$container_health" != "{}" ]; then + container_status="$(echo "$container_health" | jq -r '.Status')" + if [ "$container_status" == "healthy" ]; then + echo "Container $container_name is healthy" + return 0 + fi + fi + + retry_count=$((retry_count + 1)) + echo "Waiting for container $container_name to become healthy (attempt $retry_count of $max_retries)..." + sleep "$retry_interval" + done + + echo "Timeout reached, container $container_name is not healthy" + return 1 +} + +# Install tool to create torrent files. +# It's needed by some tests to generate and parse test torrent files. +cargo install imdl || exit 1 + +# Install app (no docker) that will run the test suite against the E2E testing +# environment (in docker). +cp .env.local .env || exit 1 + +# TEST USING MYSQL +echo "Running E2E tests using MySQL ..." + +# Start E2E testing environment +./contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh || exit 1 + +wait_for_container_to_be_healthy torrust-mysql-1 10 3 +# todo: implement healthchecks for tracker and index and wait until they are healthy +#wait_for_container torrust-tracker-1 10 3 +#wait_for_container torrust-idx-back-1 10 3 +sleep 20s + +# Just to make sure that everything is up and running +docker ps + +# Install MySQL database for the index +./contrib/dev-tools/container/e2e/mysql/install.sh || exit 1 + +# Run E2E tests with shared app instance +TORRUST_INDEX_E2E_SHARED=true TORRUST_INDEX_E2E_PATH_CONFIG="./share/default/config/index.container.mysql.toml" cargo test || exit 1 + +# Stop E2E testing environment +./contrib/dev-tools/container/e2e/mysql/e2e-env-down.sh || exit 1 + diff --git a/contrib/dev-tools/container/e2e/sqlite/e2e-env-up.sh b/contrib/dev-tools/container/e2e/sqlite/e2e-env-up.sh index e5c67632..7eb7f16f 100755 --- a/contrib/dev-tools/container/e2e/sqlite/e2e-env-up.sh +++ b/contrib/dev-tools/container/e2e/sqlite/e2e-env-up.sh @@ -4,11 +4,9 @@ TORRUST_INDEX_CONFIG=$(cat ./share/default/config/index.container.sqlite3.toml) docker compose build USER_ID=${USER_ID:-1000} \ - # Index TORRUST_INDEX_CONFIG=$(cat ./share/default/config/index.container.sqlite3.toml) \ TORRUST_INDEX_DATABASE_DRIVER="sqlite3" \ TORRUST_INDEX_TRACKER_API_TOKEN="MyAccessToken" \ - # Tracker TORRUST_TRACKER_CONFIG=$(cat ./share/default/config/tracker.container.sqlite3.toml) \ TORRUST_TRACKER_DATABASE_DRIVER="sqlite3" \ TORRUST_TRACKER_API_ADMIN_TOKEN="MyAccessToken" \ diff --git a/contrib/dev-tools/container/e2e/sqlite/install.sh b/contrib/dev-tools/container/e2e/sqlite/install.sh new file mode 100755 index 00000000..a1b800e6 --- /dev/null +++ b/contrib/dev-tools/container/e2e/sqlite/install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# This script is only intended to be used for E2E testing environment. + +# Generate storage directory if it does not exist +mkdir -p ./storage/index/lib/database + +# Generate the sqlite database if it does not exist +if ! [ -f "./storage/index/lib/database/sqlite3.db" ]; then + # todo: it should get the path from tracker.toml and only do it when we use sqlite + sqlite3 ./storage/index/lib/database/sqlite3.db "VACUUM;" +fi + diff --git a/contrib/dev-tools/container/e2e/run-e2e-tests.sh b/contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh similarity index 60% rename from contrib/dev-tools/container/e2e/run-e2e-tests.sh rename to contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh index 04c3b679..9506bc91 100755 --- a/contrib/dev-tools/container/e2e/run-e2e-tests.sh +++ b/contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh @@ -10,6 +10,7 @@ TORRUST_TRACKER_USER_UID=$CURRENT_USER_ID export TORRUST_IDX_BACK_USER_UID export TORRUST_TRACKER_USER_UID +# todo: remove duplicate funtion wait_for_container_to_be_healthy() { local container_name="$1" local max_retries="$2" @@ -42,7 +43,7 @@ cargo install imdl || exit 1 # Install app (no docker) that will run the test suite against the E2E testing # environment (in docker). cp .env.local .env || exit 1 -./contrib/dev-tools/init/install-local.sh || exit 1 +./contrib/dev-tools/container/e2e/sqlite/install.sh || exit 1 # TEST USING SQLITE echo "Running E2E tests using SQLite ..." @@ -60,39 +61,7 @@ sleep 20s docker ps # Run E2E tests with shared app instance -TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./share/default/config/index.container.sqlite3.toml" cargo test || exit 1 +TORRUST_INDEX_E2E_SHARED=true TORRUST_INDEX_E2E_PATH_CONFIG="./share/default/config/index.container.sqlite3.toml" cargo test || exit 1 # Stop E2E testing environment ./contrib/dev-tools/container/e2e/sqlite/e2e-env-down.sh || exit 1 - -# TEST USING MYSQL -echo "Running E2E tests using MySQL ..." - -# Start E2E testing environment -./contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh || exit 1 - -wait_for_container_to_be_healthy torrust-mysql-1 10 3 -# todo: implement healthchecks for tracker and index and wait until they are healthy -#wait_for_container torrust-tracker-1 10 3 -#wait_for_container torrust-idx-back-1 10 3 -sleep 20s - -# Just to make sure that everything is up and running -docker ps - -# Database credentials -MYSQL_USER="root" -MYSQL_PASSWORD="root_secret_password" -MYSQL_HOST="localhost" -MYSQL_DATABASE="torrust_index_e2e_testing" - -# Create the MySQL database for the index. Assumes MySQL client is installed. -echo "Creating MySQL database $MYSQL_DATABASE for for E2E testing ..." -mysql -h $MYSQL_HOST -u $MYSQL_USER -p$MYSQL_PASSWORD -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE;" - -# Run E2E tests with shared app instance -TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./share/default/config/index.container.mysql.toml" cargo test || exit 1 - -# Stop E2E testing environment -./contrib/dev-tools/container/e2e/mysql/e2e-env-down.sh || exit 1 - diff --git a/contrib/dev-tools/container/install.sh b/contrib/dev-tools/container/install.sh deleted file mode 100755 index 5493b7c0..00000000 --- a/contrib/dev-tools/container/install.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -./contrib/dev-tools/container/e2e/bin/build.sh -./contrib/dev-tools/init/install-local.sh diff --git a/contrib/dev-tools/init/install-local.sh b/contrib/dev-tools/init/install-local.sh index 2368de3b..7c37a06f 100755 --- a/contrib/dev-tools/init/install-local.sh +++ b/contrib/dev-tools/init/install-local.sh @@ -10,3 +10,4 @@ if ! [ -f "./storage/index/lib/database/sqlite3.db" ]; then # todo: it should get the path from tracker.toml and only do it when we use sqlite sqlite3 ./storage/index/lib/database/sqlite3.db "VACUUM;" fi + diff --git a/project-words.txt b/project-words.txt index 74669479..b8755b5a 100644 --- a/project-words.txt +++ b/project-words.txt @@ -68,6 +68,7 @@ rowid RUSTDOCFLAGS RUSTFLAGS rustfmt +rustversion serde sgxj singlepart diff --git a/tests/common/contexts/torrent/asserts.rs b/tests/common/contexts/torrent/asserts.rs index 261df54e..d0f1a8cf 100644 --- a/tests/common/contexts/torrent/asserts.rs +++ b/tests/common/contexts/torrent/asserts.rs @@ -15,7 +15,10 @@ pub fn assert_expected_torrent_details(torrent: &TorrentDetails, expected_torren ("info_hash", torrent.info_hash == expected_torrent.info_hash), ("title", torrent.title == expected_torrent.title), ("description", torrent.description == expected_torrent.description), - ("category.category_id", torrent.category.id == expected_torrent.category.id), + ( + "category.category_id", + torrent.category.category_id == expected_torrent.category.category_id, + ), ("category.name", torrent.category.name == expected_torrent.category.name), ("file_size", torrent.file_size == expected_torrent.file_size), ("seeders", torrent.seeders == expected_torrent.seeders), diff --git a/tests/common/contexts/torrent/responses.rs b/tests/common/contexts/torrent/responses.rs index b1ef0882..a6fd4946 100644 --- a/tests/common/contexts/torrent/responses.rs +++ b/tests/common/contexts/torrent/responses.rs @@ -74,9 +74,19 @@ pub struct TorrentDetails { pub encoding: Option, } +#[rustversion::stable] #[derive(Deserialize, PartialEq, Debug)] pub struct Category { - pub id: CategoryId, + pub category_id: CategoryId, // todo: rename to `id` + pub name: String, + pub num_torrents: u64, +} + +#[rustversion::nightly] +#[derive(Deserialize, PartialEq, Debug)] +#[allow(clippy::struct_field_names)] +pub struct Category { + pub category_id: CategoryId, // todo: rename to `id` pub name: String, pub num_torrents: u64, } diff --git a/tests/e2e/environment.rs b/tests/e2e/environment.rs index 73652725..1c20ca26 100644 --- a/tests/e2e/environment.rs +++ b/tests/e2e/environment.rs @@ -114,7 +114,7 @@ impl TestEnv { return match maybe_db_driver { Ok(db_driver) => match db_driver { - database::Driver::Sqlite3 => Some(db_path), + database::Driver::Sqlite3 => Some(Self::overwrite_sqlite_path(&db_path, "./storage/index/lib")), database::Driver::Mysql => Some(Self::overwrite_mysql_host(&db_path, "localhost")), }, Err(_) => None, @@ -127,7 +127,26 @@ impl TestEnv { } } - /// It overrides the "Host" in a `SQLx` database connection URL. For example: + /// It overrides the `SQLite` file path in a `SQLx` database connection URL. + /// For example: + /// + /// For: + /// + /// `sqlite:///var/lib/torrust/index/database/sqlite3.db?mode=rwc`. + /// + /// It changes the `mysql` host name to `localhost`: + /// + /// `sqlite://./storage/index/lib/database/sqlite3.db?mode=rwc`. + /// + /// For E2E tests, we use docker compose. Inside the container, the + /// `SQLite` file path is not the same as the host path. + fn overwrite_sqlite_path(db_path: &str, host_path: &str) -> String { + // todo: inject value with env var + db_path.replace("/var/lib/torrust/index", host_path) + } + + /// It overrides the "Host" in a `SQLx` database connection URL. + /// For example: /// /// For: /// @@ -138,10 +157,11 @@ impl TestEnv { /// `mysql://root:root_secret_password@localhost:3306/torrust_index_e2e_testing`. /// /// For E2E tests, we use docker compose, internally the index connects to - /// the database using the "mysql" host, which is the docker compose service - /// name, but tests connects directly to the localhost since the `MySQL` - /// is exposed to the host. + /// the `MySQL` database using the "mysql" host, which is the docker compose + /// service name, but tests connects directly to the localhost since the + /// `MySQL` is exposed to the host. fn overwrite_mysql_host(db_path: &str, new_host: &str) -> String { + // todo: inject value with env var db_path.replace("@mysql:", &format!("@{new_host}:")) } diff --git a/tests/e2e/mod.rs b/tests/e2e/mod.rs index 07c53151..ff350454 100644 --- a/tests/e2e/mod.rs +++ b/tests/e2e/mod.rs @@ -4,14 +4,14 @@ //! against an in-process server (isolated). //! //! If you want to run the tests against an out-of-process server, you need to -//! set the environment variable `TORRUST_IDX_BACK_E2E_SHARED` to `true`. +//! set the environment variable `TORRUST_INDEX_E2E_SHARED` to `true`. //! //! > **NOTICE**: The server must be running before running the tests. The //! server url is hardcoded to `http://localhost:3001` for now. We are planning //! to make it configurable in the future via a environment variable. //! //! ```text -//! TORRUST_IDX_BACK_E2E_SHARED=true cargo test +//! TORRUST_INDEX_E2E_SHARED=true cargo test //! ``` //! //! If you want to run the tests against an isolated server, you need to execute diff --git a/tests/e2e/web/api/v1/contexts/torrent/contract.rs b/tests/e2e/web/api/v1/contexts/torrent/contract.rs index 59f487b3..4b590226 100644 --- a/tests/e2e/web/api/v1/contexts/torrent/contract.rs +++ b/tests/e2e/web/api/v1/contexts/torrent/contract.rs @@ -182,7 +182,7 @@ mod for_guests { title: test_torrent.index_info.title.clone(), description: test_torrent.index_info.description, category: Category { - id: software_predefined_category_id(), + category_id: software_predefined_category_id(), name: test_torrent.index_info.category, num_torrents: 19, // Ignored in assertion },