Add a metadata service to store file metadata

Signed-off-by: Carl Schwan <[email protected]>
nextcloud · Apr 13, 2022 · 43d910a · 43d910a
1 parent d752dca
commit 43d910a
Show file tree

Hide file tree

Showing 39 changed files with 897 additions and 58 deletions.
diff --git a/3rdparty b/3rdparty
diff --git a/apps/dav/composer/autoload.php b/apps/dav/composer/autoload.php
@@ -2,6 +2,11 @@
 
 // autoload.php @generated by Composer
 
+if (PHP_VERSION_ID < 50600) {
+    echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
+    exit(1);
+}
+
 require_once __DIR__ . '/composer/autoload_real.php';
 
 return ComposerAutoloaderInitDAV::getLoader();
diff --git a/apps/dav/composer/composer/autoload_real.php b/apps/dav/composer/composer/autoload_real.php
@@ -27,7 +27,7 @@ public static function getLoader()
         spl_autoload_unregister(array('ComposerAutoloaderInitDAV', 'loadClassLoader'));
 
         require __DIR__ . '/autoload_static.php';
-        \Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader)();
+        call_user_func(\Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader));
 
         $loader->setClassMapAuthoritative(true);
         $loader->register(true);

diff --git a/apps/dav/composer/composer/installed.php b/apps/dav/composer/composer/installed.php
@@ -5,7 +5,7 @@
         'type' => 'library',
         'install_path' => __DIR__ . '/../',
         'aliases' => array(),
-        'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
+        'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
         'name' => '__root__',
         'dev' => false,
     ),
@@ -16,7 +16,7 @@
             'type' => 'library',
             'install_path' => __DIR__ . '/../',
             'aliases' => array(),
-            'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
+            'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
             'dev_requirement' => false,
         ),
     ),

diff --git a/apps/dav/lib/Connector/Sabre/Directory.php b/apps/dav/lib/Connector/Sabre/Directory.php
@@ -34,6 +34,8 @@
 
 use OC\Files\Mount\MoveableMount;
 use OC\Files\View;
+use OC\Metadata\FileMetadata;
+use OC\Metadata\MetadataGroup;
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
 use OCA\DAV\Connector\Sabre\Exception\Forbidden;
 use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
@@ -73,6 +75,9 @@ class Directory extends \OCA\DAV\Connector\Sabre\Node implements \Sabre\DAV\ICol
 	 */
 	private $tree;
 
+	/** @var array<string, array<int, FileMetadata>> */
+	private array $metadata = [];
+
 	/**
 	 * Sets up the node, expects a full path name
 	 *

diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php
@@ -43,6 +43,7 @@
 use OC\Files\Filesystem;
 use OC\Files\Stream\HashWrapper;
 use OC\Files\View;
+use OC\Metadata\FileMetadata;
 use OCA\DAV\AppInfo\Application;
 use OCA\DAV\Connector\Sabre\Exception\EntityTooLarge;
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
@@ -80,6 +81,9 @@ class File extends Node implements IFile {
 
 	protected IL10N $l10n;
 
+	/** @var array<string, FileMetadata> */
+	private array $metadata = [];
+
 	/**
 	 * Sets up the node, expects a full path name
 	 *
@@ -757,4 +761,16 @@ public function hash(string $type) {
 	public function getNode(): \OCP\Files\File {
 		return $this->node;
 	}
+
+	public function getMetadata(string $group): FileMetadata {
+		return $this->metadata[$group];
+	}
+
+	public function setMetadata(string $group, FileMetadata $metadata): void {
+		$this->metadata[$group] = $metadata;
+	}
+
+	public function hasMetadata(string $group) {
+		return array_key_exists($group, $this->metadata);
+	}
 }
diff --git a/apps/dav/lib/Connector/Sabre/FilesPlugin.php b/apps/dav/lib/Connector/Sabre/FilesPlugin.php
@@ -34,13 +34,15 @@
 namespace OCA\DAV\Connector\Sabre;
 
 use OC\AppFramework\Http\Request;
+use OC\Metadata\IMetadataManager;
 use OCP\Constants;
 use OCP\Files\ForbiddenException;
 use OCP\Files\StorageNotAvailableException;
 use OCP\IConfig;
 use OCP\IPreview;
 use OCP\IRequest;
 use OCP\IUserSession;
+use Psr\Log\LoggerInterface;
 use Sabre\DAV\Exception\Forbidden;
 use Sabre\DAV\Exception\NotFound;
 use Sabre\DAV\IFile;
@@ -50,6 +52,7 @@
 use Sabre\DAV\Tree;
 use Sabre\HTTP\RequestInterface;
 use Sabre\HTTP\ResponseInterface;
+use Sabre\Uri;
 
 class FilesPlugin extends ServerPlugin {
 
@@ -79,6 +82,7 @@ class FilesPlugin extends ServerPlugin {
 	public const SHARE_NOTE = '{http://nextcloud.org/ns}note';
 	public const SUBFOLDER_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-folder-count';
 	public const SUBFILE_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-file-count';
+	public const FILE_METADATA_SIZE = '{http://nextcloud.org/ns}file-metadata-size';
 
 	/**
 	 * Reference to main server object
@@ -436,6 +440,29 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
 			$propFind->handle(self::UPLOAD_TIME_PROPERTYNAME, function () use ($node) {
 				return $node->getFileInfo()->getUploadTime();
 			});
+
+			if ($this->config->getSystemValueBool('enable_file_metadata', true)) {
+				$propFind->handle(self::FILE_METADATA_SIZE, function () use ($node) {
+					if (!str_starts_with($node->getFileInfo()->getMimetype(), 'image')) {
+						return json_encode([]);
+					}
+
+					if ($node->hasMetadata('size')) {
+						$sizeMetadata = $node->getMetadata('size');
+					} else {
+						// This code path should not be called since we try to preload
+						// the metadata when loading the folder or the search results
+						// in one go
+						$metadataManager = \OC::$server->get(IMetadataManager::class);
+						$sizeMetadata = $metadataManager->fetchMetadataFor('size', [$node->getId()])[$node->getId()];
+
+						// TODO would be nice to display this in the profiler...
+						\OC::$server->get(LoggerInterface::class)->warning('Inefficient fetching of metadata');
+					}
+
+					return json_encode($sizeMetadata->getMetadata());
+				});
+			}
 		}
 
 		if ($node instanceof Directory) {
@@ -448,6 +475,32 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
 			});
 
 			$requestProperties = $propFind->getRequestedProperties();
+
+			// TODO detect dynamically which metadata groups are requested and
+			// preload all of them and not just size
+			if ($this->config->getSystemValueBool('enable_file_metadata', true)
+				&& in_array(self::FILE_METADATA_SIZE, $requestProperties, true)) {
+				// Preloading of the metadata
+				$fileIds = [];
+				foreach ($node->getChildren() as $child) {
+					/** @var \OCP\Files\Node|Node $child */
+					if (str_starts_with($child->getFileInfo()->getMimeType(), 'image/')) {
+						/** @var File $child */
+						$fileIds[] = $child->getFileInfo()->getId();
+					}
+				}
+				/** @var IMetaDataManager $metadataManager */
+				$metadataManager = \OC::$server->get(IMetadataManager::class);
+				$preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds);
+				foreach ($node->getChildren() as $child) {
+					/** @var \OCP\Files\Node|Node $child */
+					if (str_starts_with($child->getFileInfo()->getMimeType(), 'image')) {
+						/** @var File $child */
+						$child->setMetadata('size', $preloadedMetadata[$child->getFileInfo()->getId()]);
+					}
+				}
+			}
+
 			if (in_array(self::SUBFILE_COUNT_PROPERTYNAME, $requestProperties, true)
 				|| in_array(self::SUBFOLDER_COUNT_PROPERTYNAME, $requestProperties, true)) {
 				$nbFiles = 0;

diff --git a/apps/dav/lib/Files/FileSearchBackend.php b/apps/dav/lib/Files/FileSearchBackend.php
@@ -30,6 +30,7 @@
 use OC\Files\Search\SearchOrder;
 use OC\Files\Search\SearchQuery;
 use OC\Files\View;
+use OC\Metadata\IMetadataManager;
 use OCA\DAV\Connector\Sabre\CachingTree;
 use OCA\DAV\Connector\Sabre\Directory;
 use OCA\DAV\Connector\Sabre\FilesPlugin;
@@ -44,6 +45,7 @@
 use OCP\IUser;
 use OCP\Share\IManager;
 use Sabre\DAV\Exception\NotFound;
+use Sabre\DAV\INode;
 use SearchDAV\Backend\ISearchBackend;
 use SearchDAV\Backend\SearchPropertyDefinition;
 use SearchDAV\Backend\SearchResult;
@@ -88,14 +90,12 @@ public function __construct(CachingTree $tree, IUser $user, IRootFolder $rootFol
 
 	/**
 	 * Search endpoint will be remote.php/dav
-	 *
-	 * @return string
 	 */
-	public function getArbiterPath() {
+	public function getArbiterPath(): string {
 		return '';
 	}
 
-	public function isValidScope($href, $depth, $path) {
+	public function isValidScope(string $href, $depth, ?string $path): bool {
 		// only allow scopes inside the dav server
 		if (is_null($path)) {
 			return false;
@@ -109,7 +109,7 @@ public function isValidScope($href, $depth, $path) {
 		}
 	}
 
-	public function getPropertyDefinitionsForScope($href, $path) {
+	public function getPropertyDefinitionsForScope(string $href, ?string $path): array {
 		// all valid scopes support the same schema
 
 		//todo dynamically load all propfind properties that are supported
@@ -132,15 +132,44 @@ public function getPropertyDefinitionsForScope($href, $path) {
 			new SearchPropertyDefinition(FilesPlugin::OWNER_DISPLAY_NAME_PROPERTYNAME, false, true, false),
 			new SearchPropertyDefinition(FilesPlugin::DATA_FINGERPRINT_PROPERTYNAME, false, true, false),
 			new SearchPropertyDefinition(FilesPlugin::HAS_PREVIEW_PROPERTYNAME, false, true, false, SearchPropertyDefinition::DATATYPE_BOOLEAN),
+			new SearchPropertyDefinition(FilesPlugin::FILE_METADATA_SIZE, false, true, false, SearchPropertyDefinition::DATATYPE_STRING),
 			new SearchPropertyDefinition(FilesPlugin::FILEID_PROPERTYNAME, false, true, false, SearchPropertyDefinition::DATATYPE_NONNEGATIVE_INTEGER),
 		];
 	}
 
+	/**
+	 * @param INode[] $nodes
+	 * @param string[] $requestProperties
+	 */
+	public function preloadPropertyFor(array $nodes, array $requestProperties): void {
+		if (in_array(FilesPlugin::FILE_METADATA_SIZE, $requestProperties, true)) {
+			// Preloading of the metadata
+			$fileIds = [];
+			foreach ($nodes as $node) {
+				/** @var \OCP\Files\Node|\OCA\DAV\Connector\Sabre\Node $node */
+				if (str_starts_with($node->getFileInfo()->getMimeType(), 'image/')) {
+					/** @var \OCA\DAV\Connector\Sabre\File $node */
+					$fileIds[] = $node->getFileInfo()->getId();
+				}
+			}
+			/** @var IMetaDataManager $metadataManager */
+			$metadataManager = \OC::$server->get(IMetadataManager::class);
+			$preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds);
+			foreach ($nodes as $node) {
+				/** @var \OCP\Files\Node|\OCA\DAV\Connector\Sabre\Node $node */
+				if (str_starts_with($node->getFileInfo()->getMimeType(), 'image/')) {
+					/** @var \OCA\DAV\Connector\Sabre\File $node */
+					$node->setMetadata('size', $preloadedMetadata[$node->getFileInfo()->getId()]);
+				}
+			}
+		}
+	}
+
 	/**
 	 * @param Query $search
 	 * @return SearchResult[]
 	 */
-	public function search(Query $search) {
+	public function search(Query $search): array {
 		if (count($search->from) !== 1) {
 			throw new \InvalidArgumentException('Searching more than one folder is not supported');
 		}

diff --git a/apps/dav/lib/Files/LazySearchBackend.php b/apps/dav/lib/Files/LazySearchBackend.php
@@ -22,6 +22,7 @@
  */
 namespace OCA\DAV\Files;
 
+use Sabre\DAV\INode;
 use SearchDAV\Backend\ISearchBackend;
 use SearchDAV\Query\Query;
 
@@ -35,35 +36,38 @@ public function setBackend(ISearchBackend $backend) {
 		$this->backend = $backend;
 	}
 
-	public function getArbiterPath() {
+	public function getArbiterPath(): string {
 		if ($this->backend) {
 			return $this->backend->getArbiterPath();
 		} else {
 			return '';
 		}
 	}
 
-	public function isValidScope($href, $depth, $path) {
+	public function isValidScope(string $href, $depth, ?string $path): bool {
 		if ($this->backend) {
 			return $this->backend->getArbiterPath();
-		} else {
-			return false;
 		}
+		return false;
 	}
 
-	public function getPropertyDefinitionsForScope($href, $path) {
+	public function getPropertyDefinitionsForScope(string $href, ?String $path): array {
 		if ($this->backend) {
 			return $this->backend->getPropertyDefinitionsForScope($href, $path);
-		} else {
-			return [];
 		}
+		return [];
 	}
 
-	public function search(Query $query) {
+	public function search(Query $query): array {
 		if ($this->backend) {
 			return $this->backend->search($query);
-		} else {
-			return [];
+		}
+		return [];
+	}
+
+	public function preloadPropertyFor(array $nodes, array $requestProperties): void {
+		if ($this->backend) {
+			$this->backend->preloadPropertyFor($nodes, $requestProperties);
 		}
 	}
 }
diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml
@@ -622,9 +622,6 @@
     <UndefinedFunction occurrences="1">
       <code>\Sabre\Uri\split($sourceNode-&gt;getPath())</code>
     </UndefinedFunction>
-    <UndefinedInterfaceMethod occurrences="1">
-      <code>$info</code>
-    </UndefinedInterfaceMethod>
   </file>
   <file src="apps/dav/lib/Connector/Sabre/FakeLockerPlugin.php">
     <TooManyArguments occurrences="1">
@@ -829,10 +826,6 @@
     </InvalidScalarArgument>
   </file>
   <file src="apps/dav/lib/Files/FileSearchBackend.php">
-    <InvalidArgument occurrences="2">
-      <code>$argument</code>
-      <code>$operator-&gt;arguments</code>
-    </InvalidArgument>
     <InvalidReturnStatement occurrences="1">
       <code>$value</code>
     </InvalidReturnStatement>
@@ -842,9 +835,6 @@
     <ParamNameMismatch occurrences="1">
       <code>$search</code>
     </ParamNameMismatch>
-    <UndefinedDocblockClass occurrences="1">
-      <code>$operator-&gt;arguments[0]-&gt;name</code>
-    </UndefinedDocblockClass>
     <UndefinedPropertyFetch occurrences="1">
       <code>$operator-&gt;arguments[0]-&gt;name</code>
     </UndefinedPropertyFetch>
@@ -858,9 +848,7 @@
     <InvalidReturnStatement occurrences="1">
       <code>$this-&gt;backend-&gt;getArbiterPath()</code>
     </InvalidReturnStatement>
-    <InvalidReturnType occurrences="1">
-      <code>isValidScope</code>
-    </InvalidReturnType>
+    <InvalidReturnType occurrences="1"/>
   </file>
   <file src="apps/dav/lib/Files/RootCollection.php">
     <UndefinedFunction occurrences="1">

diff --git a/config/config.sample.php b/config/config.sample.php
@@ -2125,4 +2125,15 @@
  * Defaults to ``true``
  */
 'profile.enabled' => true,
+
+/**
+ * Enable file metadata collection
+ *
+ * This is helpful for the mobile clients and will enable a few optimization in
+ * the future for the preview generation.
+ *
+ * Note that when enabled, this data will be stored in the database and might increase
+ * the database storage.
+ */
+'enable_file_metadata' => true,
 ];
+5 −0		autoload.php
+1 −1		composer.json
+16 −8		composer.lock
+1 −1		composer/autoload_real.php
+16 −8		composer/installed.json
+5 −5		composer/installed.php
+1 −0		icewind/searchdav/.gitignore
+15 −0		icewind/searchdav/psalm.xml
+16 −6		icewind/searchdav/src/Backend/ISearchBackend.php
+6 −5		icewind/searchdav/src/Backend/SearchPropertyDefinition.php
+1 −2		icewind/searchdav/src/Backend/SearchResult.php
+2 −7		icewind/searchdav/src/DAV/DiscoverHandler.php
+1 −1		icewind/searchdav/src/DAV/PathHelper.php
+25 −23		icewind/searchdav/src/DAV/QueryParser.php
+16 −9		icewind/searchdav/src/DAV/SearchHandler.php
+10 −14		icewind/searchdav/src/DAV/SearchPlugin.php
+0 −1		icewind/searchdav/src/Query/Limit.php
+0 −1		icewind/searchdav/src/Query/Literal.php
+4 −3		icewind/searchdav/src/Query/Operator.php
+0 −1		icewind/searchdav/src/Query/Order.php
+2 −3		icewind/searchdav/src/Query/Query.php
+0 −1		icewind/searchdav/src/Query/Scope.php
+20 −12		icewind/searchdav/src/XML/BasicSearch.php
+3 −4		icewind/searchdav/src/XML/BasicSearchSchema.php
+3 −2		icewind/searchdav/src/XML/Limit.php
+1 −2		icewind/searchdav/src/XML/Literal.php
+4 −4		icewind/searchdav/src/XML/Operator.php
+3 −3		icewind/searchdav/src/XML/Order.php
+10 −3		icewind/searchdav/src/XML/PropDesc.php
+8 −6		icewind/searchdav/src/XML/QueryDiscoverResponse.php
+3 −2		icewind/searchdav/src/XML/Scope.php
+3 −6		icewind/searchdav/src/XML/SupportedQueryGrammar.php