Skip to content

Commit

Permalink
Merge pull request #25109 from nextcloud/external-scan
Browse files Browse the repository at this point in the history
add command to scan external storages directly
  • Loading branch information
icewind1991 authored Mar 7, 2024
2 parents b7088e7 + 42e14cc commit 5d2198b
Show file tree
Hide file tree
Showing 6 changed files with 295 additions and 74 deletions.
1 change: 1 addition & 0 deletions apps/files_external/appinfo/info.xml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ External storage can be configured using the GUI or at the command line. This se
<command>OCA\Files_External\Command\Backends</command>
<command>OCA\Files_External\Command\Verify</command>
<command>OCA\Files_External\Command\Notify</command>
<command>OCA\Files_External\Command\Scan</command>
</commands>

<settings>
Expand Down
2 changes: 2 additions & 0 deletions apps/files_external/composer/composer/autoload_classmap.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
'OCA\\Files_External\\Command\\ListCommand' => $baseDir . '/../lib/Command/ListCommand.php',
'OCA\\Files_External\\Command\\Notify' => $baseDir . '/../lib/Command/Notify.php',
'OCA\\Files_External\\Command\\Option' => $baseDir . '/../lib/Command/Option.php',
'OCA\\Files_External\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php',
'OCA\\Files_External\\Command\\StorageAuthBase' => $baseDir . '/../lib/Command/StorageAuthBase.php',
'OCA\\Files_External\\Command\\Verify' => $baseDir . '/../lib/Command/Verify.php',
'OCA\\Files_External\\Config\\ConfigAdapter' => $baseDir . '/../lib/Config/ConfigAdapter.php',
'OCA\\Files_External\\Config\\ExternalMountPoint' => $baseDir . '/../lib/Config/ExternalMountPoint.php',
Expand Down
2 changes: 2 additions & 0 deletions apps/files_external/composer/composer/autoload_static.php
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ class ComposerStaticInitFiles_External
'OCA\\Files_External\\Command\\ListCommand' => __DIR__ . '/..' . '/../lib/Command/ListCommand.php',
'OCA\\Files_External\\Command\\Notify' => __DIR__ . '/..' . '/../lib/Command/Notify.php',
'OCA\\Files_External\\Command\\Option' => __DIR__ . '/..' . '/../lib/Command/Option.php',
'OCA\\Files_External\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php',
'OCA\\Files_External\\Command\\StorageAuthBase' => __DIR__ . '/..' . '/../lib/Command/StorageAuthBase.php',
'OCA\\Files_External\\Command\\Verify' => __DIR__ . '/..' . '/../lib/Command/Verify.php',
'OCA\\Files_External\\Config\\ConfigAdapter' => __DIR__ . '/..' . '/../lib/Config/ConfigAdapter.php',
'OCA\\Files_External\\Config\\ExternalMountPoint' => __DIR__ . '/..' . '/../lib/Config/ExternalMountPoint.php',
Expand Down
80 changes: 6 additions & 74 deletions apps/files_external/lib/Command/Notify.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,13 @@
namespace OCA\Files_External\Command;

use Doctrine\DBAL\Exception\DriverException;
use OC\Core\Command\Base;
use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException;
use OCA\Files_External\Lib\StorageConfig;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Notify\IChange;
use OCP\Files\Notify\INotifyHandler;
use OCP\Files\Notify\IRenameChange;
use OCP\Files\Storage\INotifyStorage;
use OCP\Files\Storage\IStorage;
use OCP\Files\StorageNotAvailableException;
use OCP\IDBConnection;
use OCP\IUserManager;
use Psr\Log\LoggerInterface;
Expand All @@ -49,14 +45,14 @@
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

class Notify extends Base {
class Notify extends StorageAuthBase {
public function __construct(
private GlobalStoragesService $globalService,
private IDBConnection $connection,
private LoggerInterface $logger,
private IUserManager $userManager
GlobalStoragesService $globalService,
IUserManager $userManager,
) {
parent::__construct();
parent::__construct($globalService, $userManager);
}

protected function configure(): void {
Expand Down Expand Up @@ -97,71 +93,12 @@ protected function configure(): void {
parent::configure();
}

private function getUserOption(InputInterface $input): ?string {
if ($input->getOption('user')) {
return (string)$input->getOption('user');
}

return $_ENV['NOTIFY_USER'] ?? $_SERVER['NOTIFY_USER'] ?? null;
}

private function getPasswordOption(InputInterface $input): ?string {
if ($input->getOption('password')) {
return (string)$input->getOption('password');
}

return $_ENV['NOTIFY_PASSWORD'] ?? $_SERVER['NOTIFY_PASSWORD'] ?? null;
}

protected function execute(InputInterface $input, OutputInterface $output): int {
$mount = $this->globalService->getStorage($input->getArgument('mount_id'));
if (is_null($mount)) {
$output->writeln('<error>Mount not found</error>');
[$mount, $storage] = $this->createStorage($input, $output);
if ($storage === null) {
return self::FAILURE;
}
$noAuth = false;

$userOption = $this->getUserOption($input);
$passwordOption = $this->getPasswordOption($input);

// if only the user is provided, we get the user object to pass along to the auth backend
// this allows using saved user credentials
$user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null;

try {
$authBackend = $mount->getAuthMechanism();
$authBackend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}

if ($userOption) {
$mount->setBackendOption('user', $userOption);
}
if ($passwordOption) {
$mount->setBackendOption('password', $passwordOption);
}

try {
$backend = $mount->getBackend();
$backend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}

try {
$storage = $this->createStorage($mount);
} catch (\Exception $e) {
$output->writeln('<error>Error while trying to create storage</error>');
if ($noAuth) {
$output->writeln('<error>Login and/or password required</error>');
}
return self::FAILURE;
}
if (!$storage instanceof INotifyStorage) {
$output->writeln('<error>Mount of type "' . $mount->getBackend()->getText() . '" does not support active update notifications</error>');
return self::FAILURE;
Expand Down Expand Up @@ -189,11 +126,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
return self::SUCCESS;
}

private function createStorage(StorageConfig $mount): IStorage {
$class = $mount->getBackend()->getStorageClass();
return new $class($mount->getBackendOptions());
}

private function markParentAsOutdated($mountId, $path, OutputInterface $output, bool $dryRun): void {
$parent = ltrim(dirname($path), '/');
if ($parent === '.') {
Expand Down
155 changes: 155 additions & 0 deletions apps/files_external/lib/Command/Scan.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
<?php

declare(strict_types=1);
/**
* @copyright Copyright (c) 2021 Robin Appelman <[email protected]>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/

namespace OCA\Files_External\Command;

use OC\Files\Cache\Scanner;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\IUserManager;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

class Scan extends StorageAuthBase {
protected float $execTime = 0;
protected int $foldersCounter = 0;
protected int $filesCounter = 0;

public function __construct(
GlobalStoragesService $globalService,
IUserManager $userManager
) {
parent::__construct($globalService, $userManager);
}

protected function configure(): void {
$this
->setName('files_external:scan')
->setDescription('Scan an external storage for changed files')
->addArgument(
'mount_id',
InputArgument::REQUIRED,
'the mount id of the mount to scan'
)->addOption(
'user',
'u',
InputOption::VALUE_REQUIRED,
'The username for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'password',
'p',
InputOption::VALUE_REQUIRED,
'The password for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'path',
'',
InputOption::VALUE_OPTIONAL,
'The path in the storage to scan',
''
);
parent::configure();
}

protected function execute(InputInterface $input, OutputInterface $output): int {
[, $storage] = $this->createStorage($input, $output);
if ($storage === null) {
return 1;
}

$path = $input->getOption('path');

$this->execTime = -microtime(true);

/** @var Scanner $scanner */
$scanner = $storage->getScanner();

$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function (string $path) use ($output) {
$output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
++$this->filesCounter;
$this->abortIfInterrupted();
});

$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function (string $path) use ($output) {
$output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
++$this->foldersCounter;
$this->abortIfInterrupted();
});

$scanner->scan($path);

$this->presentStats($output);

return 0;
}

/**
* @param OutputInterface $output
*/
protected function presentStats(OutputInterface $output): void {
// Stop the timer
$this->execTime += microtime(true);

$headers = [
'Folders', 'Files', 'Elapsed time'
];

$this->showSummary($headers, [], $output);
}

/**
* Shows a summary of operations
*
* @param string[] $headers
* @param string[] $rows
* @param OutputInterface $output
*/
protected function showSummary(array $headers, array $rows, OutputInterface $output): void {
$niceDate = $this->formatExecTime();
if (!$rows) {
$rows = [
$this->foldersCounter,
$this->filesCounter,
$niceDate,
];
}
$table = new Table($output);
$table
->setHeaders($headers)
->setRows([$rows]);
$table->render();
}


/**
* Formats microtime into a human readable format
*
* @return string
*/
protected function formatExecTime(): string {
$secs = round($this->execTime);
# convert seconds into HH:MM:SS form
return sprintf('%02d:%02d:%02d', ($secs / 3600), ($secs / 60 % 60), $secs % 60);
}
}
Loading

0 comments on commit 5d2198b

Please sign in to comment.