More code refactoring; too many to list

Author: gboudreau

greyhole

@@ -45,39 +45,35 @@ public function output() {
         $max_storage_pool_strlen = max(array_map('mb_strlen', Config::storagePoolDrives()));
         $cols -= $max_storage_pool_strlen + 14;
 
-        $sorted_pool_drives = sort_storage_drives_available_space();
-        $pool_drives_avail_space = array();
-        foreach ($sorted_pool_drives as $available_space => $drive) {
-            $pool_drives_avail_space[$drive] = $available_space;
-        }
+        $pool_drives_avail_space = StoragePool::get_drives_available_space();
         $target_avail_space = array_sum($pool_drives_avail_space) / count($pool_drives_avail_space);
 
         printf("\n%$max_storage_pool_strlen"."s  %s", "", "Target free space in all storage pool drives: " . bytes_to_human($target_avail_space*1024, FALSE) . "\n");
-        $num_lines+=2;
+        $num_lines += 2;
 
-        $dfs = get_free_space_in_storage_pool_drives();
         foreach (Config::storagePoolDrives() as $sp_drive) {
-            if (!isset($dfs[$sp_drive])) {
+            $df = StoragePool::get_free_space($sp_drive);
+            if (!$df) {
                 continue;
             }
 
-            $dfs[$sp_drive]['free'] -= (float) Config::get(CONFIG_MIN_FREE_SPACE_POOL_DRIVE, $sp_drive);
+            $df['free'] -= (float) Config::get(CONFIG_MIN_FREE_SPACE_POOL_DRIVE, $sp_drive);
 
-            $percent_free = $dfs[$sp_drive]['free'] / ($dfs[$sp_drive]['free'] + $dfs[$sp_drive]['used']);
+            $percent_free = $df['free'] / ($df['free'] + $df['used']);
             $cols_free = ceil($cols * $percent_free);
             $cols_used = $cols - abs($cols_free);
 
             $suffix = "\033[0m";
-            if ($dfs[$sp_drive]['free'] < $target_avail_space) {
-                $diff = $target_avail_space - $dfs[$sp_drive]['free'];
-                $percent_diff = $diff / ($dfs[$sp_drive]['free'] + $dfs[$sp_drive]['used']);
+            if ($df['free'] < $target_avail_space) {
+                $diff = $target_avail_space - $df['free'];
+                $percent_diff = $diff / ($df['free'] + $df['used']);
                 $cols_diff = round($cols * $percent_diff);
                 $cols_used -= $cols_diff;
                 $prefix = "\033[31m";
                 $sign = '-';
             } else {
-                $diff = $dfs[$sp_drive]['free'] - $target_avail_space;
-                $percent_diff = $diff / ($dfs[$sp_drive]['free'] + $dfs[$sp_drive]['used']);
+                $diff = $df['free'] - $target_avail_space;
+                $percent_diff = $diff / ($df['free'] + $df['used']);
                 $cols_diff = round($cols * $percent_diff);
                 $cols_free -= $cols_diff;
                 $prefix = "\033[32m";

includes/CLI/BalanceStatusCliRunner.php

@@ -157,7 +157,7 @@ private function getRunner() {
         } else {
             if ($this->actionCmd->getLongOpt() != 'test-config') {
                 // Those will be tested in TestCliRunner
-                process_config();
+                ConfigHelper::test();
                 $retry_until_successful = ( $this->actionCmd->getLongOpt() == 'boot-init' );
                 DB::connect($retry_until_successful);
             }

includes/CLI/CommandLineHelper.php

@@ -41,7 +41,7 @@ private function isGoing() {
 
     public function run() {
         // Removing this drive here will insure it won't be used for new files while we're moving files away, and that it can later be replaced.
-        remove_drive_definition($this->drive);
+        StoragePool::remove_drive($this->drive);
 
         if ($this->isGoing()) {
             file_put_contents($this->drive . "/.greyhole_used_this", "Flag to prevent Greyhole from thinking this drive disappeared for no reason...");
@@ -132,7 +132,7 @@ protected static function check_going_dir($path, $share, $going_drive) {
             } else {
                 $file_path = trim(mb_substr($path, mb_strlen("$going_drive/$share")+1), '/');
                 $file_metafiles = array();
-                $file_copies_inodes = get_file_inodes($share, $file_path, $filename, $file_metafiles, TRUE);
+                $file_copies_inodes = StoragePool::get_file_copies_inodes($share, $file_path, $filename, $file_metafiles, TRUE);
                 if (count($file_copies_inodes) == 0) {
                     Log::debug("Found a file, $full_path, that has no other copies on other drives. Removing $going_drive would make that file disappear! Will create extra copies now.");
                     echo ".";

includes/CLI/GoneCliRunner.php

@@ -22,7 +22,7 @@
 
 class ReplaceCliRunner extends AbstractPoolDriveCliRunner {
     public function run() {
-        remove_drive_definition($this->drive);
+        StoragePool::remove_drive($this->drive);
 
         Log::info("Storage pool drive $this->drive has been marked replaced. The Greyhole daemon will now be restarted to allow it to use this new drive.");
         $this->log("Storage pool drive $this->drive has been marked replaced. The Greyhole daemon will now be restarted to allow it to use this new drive.");

includes/CLI/ReplaceCliRunner.php

@@ -38,10 +38,10 @@ public function run() {
             'potential_available_space' => 0
         );
 
-        $dfs = get_free_space_in_storage_pool_drives();
         $stats = array();
         foreach (Config::storagePoolDrives() as $sp_drive) {
-            if (!isset($dfs[$sp_drive]) || !file_exists($sp_drive)) {
+            $df = StoragePool::get_free_space($sp_drive);
+            if (!$df || !file_exists($sp_drive)) {
                 $stats[$sp_drive] = (object) array();
                 continue;
             }
@@ -63,7 +63,7 @@ public function run() {
                 }
             }
 
-            $free_space = $dfs[$sp_drive]['free'];
+            $free_space = $df['free'];
 
             $trash_path = clean_dir("$sp_drive/.gh_trash");
             if (!file_exists($trash_path)) {

includes/CLI/StatsCliRunner.php

@@ -22,7 +22,7 @@
 
 class TestCliRunner extends AbstractCliRunner {
     public function run() {
-        process_config();
+        ConfigHelper::test();
         DB::connect();
         DB::repairTables();
         MigrationHelper::convertStoragePoolDrivesTagFiles();

includes/CLI/TestCliRunner.php

@@ -563,6 +563,22 @@ private static function assert($check, $error_message, $event_code) {
             Log::critical($error_message, $event_code);
         }
     }
+
+    public static function test() {
+        while (!ConfigHelper::parse()) {
+            // Invalid config file; either it's missing storage_pool_drive, or it contains a share that isn't in smb.conf
+            if (SystemHelper::is_amahi() && Log::actionIs(ACTION_DAEMON)) {
+                // If running on Amahi, loop until the config works.
+                // User might configure Greyhole later, and they don't want to show Greyhole 'offline' until then. Those users are easy to confuse! ;)
+                sleep(600); // 10 minutes
+            } else {
+                // Otherwise, die.
+                Log::critical("Config file parsing failed. Exiting.", Log::EVENT_CODE_CONFIG_FILE_PARSING_FAILED);
+            }
+        }
+        // Config is OK; go on!
+    }
+
 }
 
 final class Config {
@@ -694,6 +710,62 @@ public static function add($share, $name, $value, $index=NULL) {
         }
         self::$shares_config[$share] = $config;
     }
+
+    public static function getNumCopies($share) {
+        $num_copies = static::get($share, CONFIG_NUM_COPIES);
+        if (!$num_copies) {
+            Log::warn("Found a task on a share ($share) that disappeared from " . ConfigHelper::$config_file . ". Skipping.", Log::EVENT_CODE_TASK_FOR_UNKNOWN_SHARE);
+            return -1;
+        }
+        if ($num_copies < 1) {
+            $num_copies = 1;
+        }
+        $max_copies = 0;
+        foreach (Config::storagePoolDrives() as $sp_drive) {
+            if (StoragePool::is_pool_drive($sp_drive)) {
+                $max_copies++;
+            }
+        }
+        if ($num_copies > $max_copies) {
+            $num_copies = $max_copies;
+        }
+        return $num_copies;
+    }
+
+    public static function getShareOptions($full_path) {
+        $share = FALSE;
+        $landing_zone = '';
+        foreach (SharesConfig::getShares() as $share_name => $share_options) {
+            $lz = $share_options[CONFIG_LANDING_ZONE];
+            if (string_starts_with($full_path, $lz) && mb_strlen($lz) > mb_strlen($landing_zone)) {
+                $landing_zone = $lz;
+                $share = $share_options;
+            }
+        }
+        return $share;
+    }
+
+    function getShareOptionsFromDrive($full_path, $sp_drive) {
+        $landing_zone = '';
+        $share = FALSE;
+        foreach (SharesConfig::getShares() as $share_name => $share_options) {
+            $lz = $share_options[CONFIG_LANDING_ZONE];
+            $metastore = Metastores::get_metastore_from_path($full_path);
+            if ($metastore !== FALSE) {
+                if (string_starts_with($full_path, "$metastore/$share_name") && mb_strlen($lz) > mb_strlen($landing_zone)) {
+                    $landing_zone = $lz;
+                    $share = $share_options;
+                }
+            } else {
+                if (string_starts_with($full_path, "$sp_drive/$share_name") && mb_strlen($lz) > mb_strlen($landing_zone)) {
+                    $landing_zone = $lz;
+                    $share = $share_options;
+                }
+            }
+        }
+        return $share;
+    }
+
 }
 
 ?>

includes/ConfigHelper.php

@@ -118,16 +118,53 @@ public static function setNextTask($task) {
         array_unshift(static::getInstance()->next_tasks, $task);
     }
 
-    public static function lockFile($idx, $locked_by) {
-        static::getInstance()->locked_files[$idx] = $locked_by;
+    private function lockFile($idx, $locked_by) {
+        $this->locked_files[$idx] = $locked_by;
     }
 
-    public static function isFileLocked($full_path) {
+    public static function isFileLocked($share, $full_path) {
         $db_spool = static::getInstance();
-        if (isset($db_spool->locked_files[$full_path])) {
-            return $db_spool->locked_files[$full_path];
+        $idx = clean_dir("$share/$full_path");
+        if (isset($db_spool->locked_files[$idx])) {
+            return $db_spool->locked_files[$idx];
         }
-        return FALSE;
+
+        if (Config::get(CONFIG_CHECK_FOR_OPEN_FILES) === FALSE) {
+            Log::debug("  Skipping open file (lock) check.");
+            return FALSE;
+        }
+
+        $landing_zone = get_share_landing_zone($share);
+        if (!$landing_zone) {
+            return FALSE;
+        }
+
+        $real_fullpath = "$landing_zone/$full_path";
+
+        $result = gh_is_file_locked($real_fullpath);
+        if ($result !== FALSE) {
+            $db_spool->lockFile($idx, $result);
+            return $result;
+        }
+
+        $query = "SELECT * FROM tasks WHERE complete = 'no' AND action = 'write' AND share = :share AND full_path = :full_path LIMIT 1";
+        $params = array('share' => $share, 'full_path' => $full_path);
+        $row = DB::getFirst($query, $params);
+        if ($row === FALSE) {
+            return FALSE;
+        }
+
+        // Locked, according to DB... But maybe it's not really locked?
+        if (!gh_file_exists($real_fullpath)) {
+            // File doesn't exists anymore... It can't be really locked... Let's assume this is just Samba that 'forgot' to close the file handle.
+            $query = "UPDATE tasks SET complete = 'yes' WHERE complete = 'no' AND action = 'write' AND share = :share AND full_path = :full_path";
+            DB::execute($query, $params);
+            return FALSE;
+        }
+
+        $result = 'samba-vfs-writer';
+        $db_spool->lockFile($idx, $result);
+        return $result;
     }
 
     public function execute_next_task() {
@@ -200,7 +237,7 @@ public function execute_next_task() {
         }
 
         if ($task->complete == 'written') {
-            if (should_ignore_file($task->share, $task->full_path)) {
+            if ($task->should_ignore_file()) {
                 $this->archive_task($task->id);
                 return;
             }
@@ -224,7 +261,7 @@ public function execute_next_task() {
 
             if (!empty($this->written_files[clean_dir("$task->share/$task->full_path")])) {
                 Log::debug("  File is still being written to (" . bytes_to_human($filesize, FALSE) . "). Postponing.");
-                static::lockFile(clean_dir("$task->share/$task->full_path"), 'samba-bytes-writer');
+                $this->lockFile(clean_dir("$task->share/$task->full_path"), 'samba-bytes-writer');
                 $this->locked_shares[$task->share] = TRUE;
                 return;
             }

includes/DBSpool.php

@@ -21,7 +21,7 @@
 class DaemonRunner extends AbstractRunner {
 
 	public static $was_idle = TRUE;
-	
+
 	public function run() {
 		// Prevent multiple daemons from running simultaneously
 		if (self::isRunning()) {
@@ -57,14 +57,14 @@ private static function isRunning() {
 	private static function getPID() {
         return exec('ps ax | grep "greyhole --daemon\|greyhole -D" | grep -v grep | grep -v bash | grep -v "sudo" | grep -v ' . getmypid() . ' | awk "{print \$1}"');
 	}
-    
+
 	private function initialize() {
 		// Check the database tables, and repair them if needed.
         DB::repairTables();
-		
+
 		// Creates a GUID (if one doesn't exist); will be used to uniquely identify this Greyhole install, when reporting anonymous usage to greyhole.net
         GetGUIDCliRunner::setUniqID();
-		
+
 		// Terminology changed (attic > trash, graveyard > metadata store, tombstones > metadata files); this requires filesystem & database changes.
         MigrationHelper::terminologyConversion();
 
@@ -73,20 +73,20 @@ private function initialize() {
 
 		// We backup the database settings to disk, in order to be able to restore them if the database is lost.
         Settings::backup();
-		
+
 		// Check that the Greyhole VFS module used by Samba is the correct one for the current Samba version. This is needed when Samba is updated to a new major version after Greyhole is installed.
         SambaUtils::samba_check_vfs();
 
         // Check if the in-memory spool folder exists, and if no, create it and mount a tmpfs there. VFS will write there recvfile (etc.) operations.
 		SambaSpool::create_mem_spool();
-		
+
 		// Process the spool directory, and insert each task found there into the database.
 		SambaSpool::parse_samba_spool();
-		
+
         // Create the dfree cache folder, if it doesn't exist
-        gh_mkdir('/var/cache/greyhole-dfree', (object) array('fileowner' => 0, 'filegroup' => 0, 'fileperms' => (int) base_convert("0777", 8, 10)));
+        gh_mkdir('/var/cache/greyhole-dfree', NULL, (object) array('fileowner' => 0, 'filegroup' => 0, 'fileperms' => (int) base_convert("0777", 8, 10)));
 	}
-	
+
 	public function finish($returnValue = 0) {
 		// The daemon should never finish; it will be killed by the init script.
 		// Not that it can reach finish() anyway, since it's in an infinite while(TRUE) loop in run()... :)

includes/DaemonRunner.php

@@ -149,7 +149,7 @@ public static function get_metastore_from_path($path) {
     public static function get_metastores_from_storage_volume($storage_volume) {
         $volume_metastores = array();
         foreach (static::get_metastores() as $metastore) {
-            if (get_storage_volume_from_path($metastore) == $storage_volume) {
+            if (StoragePool::getDriveFromPath($metastore) == $storage_volume) {
                 $volume_metastores[] = $metastore;
             }
         }
@@ -243,7 +243,7 @@ public static function get_metafiles_for_file($share, $path, $filename=NULL, $lo
         foreach ($metafiles as $key => $metafile) {
             $valid_path = FALSE;
 
-            $drive = get_storage_volume_from_path($metafile->path);
+            $drive = StoragePool::getDriveFromPath($metafile->path);
             if ($drive !== FALSE) {
                 $valid_path = TRUE;
             }
@@ -287,11 +287,11 @@ public static function create_metafiles($share, $full_path, $num_copies_required
             }
 
             // Check free space!
-            $dfs = get_free_space_in_storage_pool_drives();
-            if (!isset($dfs[$sp_drive])) {
+            $df = StoragePool::get_free_space($sp_drive);
+            if (!$df) {
                 $free_space = 0;
             } else {
-                $free_space = $dfs[$sp_drive]['free'];
+                $free_space = $df['free'];
             }
             if ($free_space <= $filesize/1024) {
                 $metafile->state = Metafile::STATE_GONE;
@@ -312,11 +312,11 @@ public static function create_metafiles($share, $full_path, $num_copies_required
 
         // Select drives that have enough free space for this file
         if ($num_ok < $num_copies_required) {
-            $local_target_drives = order_target_drives($filesize/1024, FALSE, $share, $path, '  ');
+            $target_drives = StoragePool::choose_target_drives($filesize/1024, FALSE, $share, $path, '  ');
         }
         /** @noinspection PhpUndefinedVariableInspection */
-        while ($num_ok < $num_copies_required && count($local_target_drives) > 0) {
-            $sp_drive = array_shift($local_target_drives);
+        while ($num_ok < $num_copies_required && count($target_drives) > 0) {
+            $sp_drive = array_shift($target_drives);
             $clean_target_full_path = clean_dir("$sp_drive/$share/$full_path");
             // Don't use drives that already have a copy
             if (isset($metafiles[$clean_target_full_path])) {
@@ -360,7 +360,7 @@ public static function save_metafiles($share, $path, $filename, $metafiles) {
             $data_filepath = clean_dir("$metastore/$share/$path");
             $has_metafile = FALSE;
             foreach ($metafiles as $metafile) {
-                if (get_storage_volume_from_path($metafile->path) == $sp_drive && StoragePool::is_pool_drive($sp_drive)) {
+                if (StoragePool::getDriveFromPath($metafile->path) == $sp_drive && StoragePool::is_pool_drive($sp_drive)) {
                     gh_mkdir($data_filepath, get_share_landing_zone($share) . "/$path");
                     Log::debug("    Saving metadata in " . clean_dir("$data_filepath/$filename"));
                     if (is_dir("$data_filepath/$filename")) {