feat(processing): follow symlinks by default

Makefile

@@ -51,7 +51,7 @@ dev-spis: ${DATABASE} ${MEDIA_DIR} ${THUMBNAIL_DIR}
 
 .PHONY: dev-nginx
 dev-nginx: ${DATABASE} ${MEDIA_DIR} ${THUMBNAIL_DIR}
-	bash -c 'cargo run -q -- template nginx --full > /tmp/nginx.conf && nginx -g "daemon off;" -c /tmp/nginx.conf'
+	bash -c 'RUST_LOG=error cargo run -q -- template nginx --full > /tmp/nginx.conf && nginx -g "daemon off;" -c /tmp/nginx.conf'
 
 .PHONY: dev
 dev:

README.md

@@ -86,25 +86,27 @@ Commands:
 
 Options:
       --media-dir <MEDIA_DIR>
-          Path to search for media files [env: SPIS_MEDIA_DIR=/home/gbbirkisson/repos/personal/spis/data/media] [default: ]
+          Path to search for media files [env: SPIS_MEDIA_DIR=] [default: ]
       --data-dir <DATA_DIR>
-          Path to store data [env: SPIS_DATA_DIR=/home/gbbirkisson/repos/personal/spis/data] [default: ]
+          Path to store data [env: SPIS_DATA_DIR=] [default: ]
       --processing-schedule <PROCESSING_SCHEDULE>
-          Schedule to run processing on [env: SPIS_PROCESSING_SCHEDULE=0 0 2 * * *] [default: "0 0 2 * * *"]
+          Schedule to run processing on [env: SPIS_PROCESSING_SCHEDULE=] [default: "0 0 2 * * *"]
       --processing-run-on-start
-          Run processing on start [env: SPIS_PROCESSING_RUN_ON_START=true]
+          Run processing on start [env: SPIS_PROCESSING_RUN_ON_START=]
       --api-media-path <API_MEDIA_PATH>
-          Path webserver will serve media on [env: SPIS_API_MEDIA_PATH=/assets/media] [default: /assets/media]
+          Path webserver will serve media on [env: SPIS_API_MEDIA_PATH=] [default: /assets/media]
       --api-thumbnail-path <API_THUMBNAIL_PATH>
-          Path webserver will serve thumbnails on [env: SPIS_API_THUMBNAIL_PATH=/assets/thumbnails] [default: /assets/thumbnails]
+          Path webserver will serve thumbnails on [env: SPIS_API_THUMBNAIL_PATH=] [default: /assets/thumbnails]
       --server-address <SERVER_ADDRESS>
           Listen to address [env: SPIS_SERVER_ADDRESS=]
       --server-socket <SERVER_SOCKET>
-          Listen to UNIX socket [env: SPIS_SERVER_SOCKET=/tmp/spis.sock]
+          Listen to UNIX socket [env: SPIS_SERVER_SOCKET=]
       --feature-favorite
-          Disable feature favorite [env: SPIS_FEATURE_FAVORITE=true]
+          Disable feature favorite [env: SPIS_FEATURE_FAVORITE=]
       --feature-archive
-          Disable feature archive [env: SPIS_FEATURE_ARCHIVE=true]
+          Disable feature archive [env: SPIS_FEATURE_ARCHIVE=]
+      --feature-follow-symlinks
+          Disable feature follow symlinks [env: SPIS_FEATURE_FOLLOW_SYMLINKS=]
   -h, --help
           Print help
   -V, --version

examples/docker/docker-compose.yml

@@ -9,6 +9,7 @@ services:
       SPIS_PROCESSING_RUN_ON_START: "false"
       SPIS_FEATURE_FAVORITE: "true"
       SPIS_FEATURE_ARCHIVE: "true"
+      SPIS_FEATURE_FOLLOW_SYMLINKS: "true"
       RUST_LOG: "error,spis=info"
     volumes:
       - "/tmp/spis_data:/var/lib/spis/data"

examples/systemd/spis.service

@@ -12,6 +12,7 @@ Environment="SPIS_API_THUMBNAIL_PATH=/assets/thumbnails"
 Environment="SPIS_SERVER_SOCKET=/storage/spis/data/spis.sock"
 Environment="SPIS_FEATURE_FAVORITE=true"
 Environment="SPIS_FEATURE_ARCHIVE=true"
+Environment="SPIS_FEATURE_FOLLOW_SYMLINKS=true"
 Environment="RUST_LOG=error,spis=info"
 User=www-data
 ExecStart=/usr/bin/spis

src/main.rs

@@ -25,6 +25,7 @@ fn file_exists(s: &str) -> Result<PathBuf, String> {
 /// Simple private image server
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
+#[allow(clippy::struct_excessive_bools)]
 pub struct Spis {
     /// Path to search for media files
     #[clap(long, env = "SPIS_MEDIA_DIR", default_value = "")]
@@ -65,6 +66,10 @@ pub struct Spis {
     #[clap(long, env = "SPIS_FEATURE_ARCHIVE", default_value = "true", action=ArgAction::SetFalse)]
     pub feature_archive: bool,
 
+    /// Disable feature follow symlinks
+    #[clap(long, env = "SPIS_FEATURE_FOLLOW_SYMLINKS", default_value = "true", action=ArgAction::SetFalse)]
+    pub feature_follow_symlinks: bool,
+
     #[command(subcommand)]
     command: Option<SpisCommand>,
 }
@@ -273,9 +278,13 @@ async fn run(config: Spis) -> Result<()> {
         .wrap_err("Failed to start file watcher")?;
 
     tracing::info!("Setting up file walker");
-    let job_sender =
-        pipeline::setup_filewalker(pool.clone(), config.media_dir.clone(), file_sender.clone())
-            .wrap_err("Failed to setup file walker")?;
+    let job_sender = pipeline::setup_filewalker(
+        pool.clone(),
+        config.media_dir.clone(),
+        file_sender.clone(),
+        config.feature_follow_symlinks,
+    )
+    .wrap_err("Failed to setup file walker")?;
 
     pipeline::setup_db_store(pool.clone(), media_receiver).wrap_err("Failed to setup db store")?;
 

src/pipeline.rs

@@ -7,6 +7,7 @@ use chrono::Local;
 use chrono::{DateTime, Duration, Utc};
 use color_eyre::eyre::Context;
 use color_eyre::Result;
+use notify::event::ModifyKind;
 use notify::{
     event::{AccessKind, CreateKind, EventKind},
     Config, Error, Event, RecommendedWatcher, Watcher,
@@ -87,7 +88,8 @@ pub fn setup_filewatcher(file_sender: Sender<File>) -> Result<RecommendedWatcher
                 tracing::trace!("Got file event: {:?}", event);
                 let trigger_processing = match event.kind {
                     EventKind::Create(CreateKind::File)
-                    | EventKind::Access(AccessKind::Close(_)) => true,
+                    | EventKind::Access(AccessKind::Close(_))
+                    | EventKind::Modify(ModifyKind::Name(_)) => true,
                     // EventKind::Remove(RemoveKind::File) => {
                     //     // TODO: Handle file deletions?
                     // }
@@ -115,6 +117,7 @@ pub fn setup_filewalker(
     pool: Pool<Sqlite>,
     media_dir: PathBuf,
     file_sender: Sender<File>,
+    follow_symlinks: bool,
 ) -> Result<Sender<JobTrigger>> {
     tracing::debug!("Setup file walker");
 
@@ -150,7 +153,7 @@ pub fn setup_filewalker(
 
             tracing::debug!("Start walk thread");
             tokio::task::spawn_blocking(move || {
-                walk_dir(&old_uuids, &media_dir, &file_sender);
+                walk_dir(&old_uuids, &media_dir, &file_sender, follow_symlinks);
                 if let Err(error) = walk_finished_sender.send(NOTHING) {
                     tracing::error!("Failed to trigger processing finish: {:?}", error);
                 };
@@ -226,9 +229,10 @@ fn walk_dir(
     old_uuids: &HashMap<String, Uuid>,
     media_dir: &PathBuf,
     file_sender: &Sender<(Option<Uuid>, PathBuf)>,
+    follow_symlinks: bool,
 ) {
     let mut count = 0;
-    for entry in WalkDir::new(media_dir) {
+    for entry in WalkDir::new(media_dir).follow_links(follow_symlinks) {
         count += 1;
         if count % 1000 == 0 {
             tracing::info!("Walked {} files so far ...", count);

templates/config/docker-compose.yml

@@ -9,6 +9,7 @@ services:
       SPIS_PROCESSING_RUN_ON_START: "{{ config.processing_run_on_start }}"
       SPIS_FEATURE_FAVORITE: "{{ config.feature_favorite }}"
       SPIS_FEATURE_ARCHIVE: "{{ config.feature_archive }}"
+      SPIS_FEATURE_FOLLOW_SYMLINKS: "{{ config.feature_follow_symlinks }}"
       RUST_LOG: "{{ spis_log }}"
     volumes:
       - "{{ config.data_dir.to_str().unwrap() }}:/var/lib/spis/data"

templates/config/systemd.service

@@ -18,6 +18,7 @@ Environment="SPIS_SERVER_SOCKET={{ socket }}"
 {%- endif %}
 Environment="SPIS_FEATURE_FAVORITE={{ config.feature_favorite }}"
 Environment="SPIS_FEATURE_ARCHIVE={{ config.feature_archive }}"
+Environment="SPIS_FEATURE_FOLLOW_SYMLINKS={{ config.feature_follow_symlinks }}"
 Environment="RUST_LOG={{ spis_log }}"
 User={{ spis_user }}
 ExecStart={{ spis_bin }}