Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Contains query v2 #377

Merged
merged 56 commits into from
Jan 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
fdbbb76
Add array contains support for SQL adapters
abnegate Oct 9, 2023
fc6b198
Remove dupe semicolon
abnegate Oct 9, 2023
e8295c1
Disable SQLite array contains
abnegate Oct 9, 2023
2985269
Disallow contains queries on non-array attributes
abnegate Oct 12, 2023
15ba9b1
Add test for non-array attribute
abnegate Oct 12, 2023
7122006
Commit benchmark results for comparisons
abnegate Oct 20, 2023
f68bbef
Add contains queries to benchmarks
abnegate Oct 20, 2023
893da60
lock file
fogelito Dec 25, 2023
6a55fbd
Merge branch 'main' of github.com:utopia-php/database into feat-array…
fogelito Dec 25, 2023
96c6945
Query name changes
fogelito Dec 25, 2023
32d49fa
Remove old parse tests
fogelito Dec 25, 2023
4149b7a
array changes and new test
fogelito Dec 26, 2023
26a16f6
getSupportForIndexArray
fogelito Dec 26, 2023
2ea81a1
find test
fogelito Dec 26, 2023
54f6166
tests
fogelito Dec 27, 2023
361f379
try json_overlaps
fogelito Dec 28, 2023
d441401
tests
fogelito Dec 28, 2023
5a3d56f
createIndex order test
fogelito Dec 31, 2023
cc22abd
Order tests
fogelito Jan 3, 2024
9846af4
Order tests
fogelito Jan 4, 2024
619f797
Signature change create Index
fogelito Jan 4, 2024
1ce5bdf
Change createIndex signature
fogelito Jan 4, 2024
5f4e291
Mongo
fogelito Jan 7, 2024
9ba5e82
more tests
fogelito Jan 7, 2024
b8ab877
more tests
fogelito Jan 7, 2024
6d6ff1c
tests
fogelito Jan 8, 2024
167f5bd
Merge branch 'main' of github.com:utopia-php/database into contains-v2
fogelito Jan 8, 2024
de4ccaa
lint + composer.lock
fogelito Jan 8, 2024
1a38083
Move getSQLIndex
fogelito Jan 8, 2024
f9e3f7f
Check contains with validator
fogelito Jan 8, 2024
3410db8
version upgrade
fogelito Jan 9, 2024
33fa6c8
Update phpunit.xml
fogelito Jan 14, 2024
5a832a1
Update src/Database/Validator/Index.php
fogelito Jan 14, 2024
a5d9acf
Update src/Database/Validator/Index.php
fogelito Jan 14, 2024
9424767
Update bin/tasks/load.php
fogelito Jan 14, 2024
0bb90a4
Update src/Database/Validator/Index.php
fogelito Jan 14, 2024
5431d5a
Addressing comments
fogelito Jan 14, 2024
bb018fc
Update src/Database/Database.php
fogelito Jan 14, 2024
6a32656
Update src/Database/Adapter/SQLite.php
fogelito Jan 14, 2024
16df875
Update src/Database/Validator/Index.php
fogelito Jan 14, 2024
8f13042
Mysql check cast
fogelito Jan 14, 2024
1a8a21b
Merge remote-tracking branch 'origin/contains-v2' into contains-v2
fogelito Jan 14, 2024
b0c3900
formatting
fogelito Jan 14, 2024
8c8fe43
Cannot set a length text change
fogelito Jan 14, 2024
5a04626
remove var_dumps
fogelito Jan 14, 2024
5dd561b
Test Integer array size
fogelito Jan 15, 2024
e49ac27
Default index length
fogelito Jan 16, 2024
2315c49
Update src/Database/Adapter/MariaDB.php
fogelito Jan 17, 2024
b4869c5
Update src/Database/Adapter/SQL.php
fogelito Jan 17, 2024
1b999c5
Update src/Database/Validator/Query/Filter.php
fogelito Jan 17, 2024
35c6d2c
Update src/Database/Validator/Query/Filter.php
fogelito Jan 17, 2024
2e11ad0
Addressing comments
fogelito Jan 17, 2024
a7aeed2
Merge remote-tracking branch 'origin/contains-v2' into contains-v2
fogelito Jan 17, 2024
18ba005
Addressing comments
fogelito Jan 17, 2024
db7bf05
Upper case like
fogelito Jan 18, 2024
75b75b0
Fix Mongo case sensitive
fogelito Jan 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,9 @@ mock.json
data-tests.php
loader.php
.phpunit.result.cache
/bin/view/results/
.vscode
.vscode/*
database.sql

## - Oh Wess!
Makefile
.envrc
.vscode
Expand Down
3 changes: 2 additions & 1 deletion bin/cli.php
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
<?php

// require_once __DIR__.'/init.php';
require_once '/usr/src/code/vendor/autoload.php';

use Utopia\CLI\CLI;

ini_set('memory_limit', '-1');

$cli = new CLI();

include 'tasks/load.php';
Expand Down
20 changes: 10 additions & 10 deletions bin/tasks/index.php
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@

/**
* @Example
* docker-compose exec tests bin/index --adapter=mysql --name=testing
* docker compose exec tests bin/index --adapter=mysql --name=testing
*/

$cli
->task('index')
->desc('Index mock data for testing queries')
->param('adapter', '', new Text(0), 'Database adapter', false)
->param('name', '', new Text(0), 'Name of created database.', false)
->param('adapter', '', new Text(0), 'Database adapter')
->param('name', '', new Text(0), 'Name of created database.')
->action(function ($adapter, $name) {
$namespace = '_ns';
$cache = new Cache(new NoCache());
Expand Down Expand Up @@ -74,36 +74,36 @@
$database->setDatabase($name);
$database->setNamespace($namespace);

Console::info("For query: greaterThan(created, 2010-01-01 05:00:00)', 'equal(genre,travel)");

Console::info("greaterThan('created', ['2010-01-01 05:00:00']), equal('genre', ['travel'])");
$start = microtime(true);
$database->createIndex('articles', 'createdGenre', Database::INDEX_KEY, ['created', 'genre'], [], [Database::ORDER_DESC, Database::ORDER_DESC]);
$time = microtime(true) - $start;
Console::success("{$time} seconds");

Console::info("equal('genre', ['fashion', 'finance', 'sports'])");

$start = microtime(true);
$database->createIndex('articles', 'genre', Database::INDEX_KEY, ['genre'], [], [Database::ORDER_ASC]);
$time = microtime(true) - $start;
Console::success("{$time} seconds");


Console::info("greaterThan('views', 100000)");

$start = microtime(true);
$database->createIndex('articles', 'views', Database::INDEX_KEY, ['views'], [], [Database::ORDER_DESC]);
$time = microtime(true) - $start;
Console::success("{$time} seconds");


Console::info("search('text', 'Alice')");
$start = microtime(true);
$database->createIndex('articles', 'fulltextsearch', Database::INDEX_FULLTEXT, ['text']);
$time = microtime(true) - $start;
Console::success("{$time} seconds");
});

Console::info("contains('tags', ['tag1'])");
$start = microtime(true);
$database->createIndex('articles', 'tags', Database::INDEX_KEY, ['tags']);
$time = microtime(true) - $start;
Console::success("{$time} seconds");
});

$cli
->error()
Expand Down
23 changes: 13 additions & 10 deletions bin/tasks/load.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

/**
* @Example
* docker-compose exec tests bin/load --adapter=mariadb --limit=1000 --name=testing
* docker compose exec tests bin/load --adapter=mariadb --limit=1000 --name=testing
*/

$cli
Expand All @@ -44,6 +44,7 @@
Console::info("Filling {$adapter} with {$limit} records: {$name}");

Swoole\Runtime::enableCoroutine();

switch ($adapter) {
case 'mariadb':
Co\run(function () use (&$start, $limit, $name, $namespace, $cache) {
Expand Down Expand Up @@ -85,7 +86,7 @@

// A coroutine is assigned per 1000 documents
for ($i = 0; $i < $limit / 1000; $i++) {
go(function () use ($pool, $faker, $name, $cache, $namespace) {
\go(function () use ($pool, $faker, $name, $cache, $namespace) {
$pdo = $pool->get();

$database = new Database(new MariaDB($pdo), $cache);
Expand All @@ -94,7 +95,7 @@

// Each coroutine loads 1000 documents
for ($i = 0; $i < 1000; $i++) {
addArticle($database, $faker);
createDocument($database, $faker);
}

// Reclaim resources
Expand Down Expand Up @@ -146,7 +147,7 @@

// A coroutine is assigned per 1000 documents
for ($i = 0; $i < $limit / 1000; $i++) {
go(function () use ($pool, $faker, $name, $cache, $namespace) {
\go(function () use ($pool, $faker, $name, $cache, $namespace) {
$pdo = $pool->get();

$database = new Database(new MySQL($pdo), $cache);
Expand All @@ -155,7 +156,7 @@

// Each coroutine loads 1000 documents
for ($i = 0; $i < 1000; $i++) {
addArticle($database, $faker);
createDocument($database, $faker);
}

// Reclaim resources
Expand Down Expand Up @@ -197,7 +198,7 @@

// Each coroutine loads 1000 documents
for ($i = 0; $i < 1000; $i++) {
addArticle($database, $faker);
createDocument($database, $faker);
}

$database = null;
Expand Down Expand Up @@ -233,25 +234,26 @@ function createSchema(Database $database): void
$database->create();

Authorization::setRole(Role::any()->toString());

fogelito marked this conversation as resolved.
Show resolved Hide resolved
$database->createCollection('articles', permissions: [
Permission::create(Role::any()),
Permission::read(Role::any()),
]);

$database->createAttribute('articles', 'author', Database::VAR_STRING, 256, true);
$database->createAttribute('articles', 'created', Database::VAR_DATETIME, 0, true, null, false, false, null, [], ['datetime']);
$database->createAttribute('articles', 'created', Database::VAR_DATETIME, 0, true, filters: ['datetime']);
$database->createAttribute('articles', 'text', Database::VAR_STRING, 5000, true);
$database->createAttribute('articles', 'genre', Database::VAR_STRING, 256, true);
$database->createAttribute('articles', 'views', Database::VAR_INTEGER, 0, true);
$database->createAttribute('articles', 'tags', Database::VAR_STRING, 0, true, array: true);
$database->createIndex('articles', 'text', Database::INDEX_FULLTEXT, ['text']);
}

function addArticle($database, Generator $faker): void
function createDocument($database, Generator $faker): void
{
$database->createDocument('articles', new Document([
// Five random users out of 10,000 get read access
// Three random users out of 10,000 get mutate access

'$permissions' => [
Permission::read(Role::any()),
Permission::read(Role::user($faker->randomNumber(9))),
Expand All @@ -272,6 +274,7 @@ function addArticle($database, Generator $faker): void
'created' => \Utopia\Database\DateTime::format($faker->dateTime()),
'text' => $faker->realTextBetween(1000, 4000),
'genre' => $faker->randomElement(['fashion', 'food', 'travel', 'music', 'lifestyle', 'fitness', 'diy', 'sports', 'finance']),
'views' => $faker->randomNumber(6)
'views' => $faker->randomNumber(6),
'tags' => $faker->randomElements(['short', 'quick', 'easy', 'medium', 'hard'], $faker->numberBetween(1, 5)),
]));
}
79 changes: 39 additions & 40 deletions bin/tasks/query.php
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@

/**
* @Example
* docker-compose exec tests bin/query --adapter=mariadb --limit=1000 --name=testing
* docker compose exec tests bin/query --adapter=mariadb --limit=1000 --name=testing
*/
$cli
->task('query')
->desc('Query mock data')
->param('adapter', '', new Text(0), 'Database adapter', false)
->param('name', '', new Text(0), 'Name of created database.', false)
->param('adapter', '', new Text(0), 'Database adapter')
->param('name', '', new Text(0), 'Name of created database.')
->param('limit', 25, new Numeric(), 'Limit on queried documents', true)
->action(function (string $adapter, string $name, int $limit) {
$namespace = '_ns';
Expand Down Expand Up @@ -80,113 +80,112 @@
return;
}


$faker = Factory::create();

$report = [];

$count = addRoles($faker, 1);
$count = setRoles($faker, 1);
Console::info("\n{$count} roles:");
$report[] = [
'roles' => $count,
'results' => runQueries($database, $limit)
];

$count = addRoles($faker, 100);
$count = setRoles($faker, 100);
Console::info("\n{$count} roles:");
$report[] = [
'roles' => $count,
'results' => runQueries($database, $limit)
];

$count = addRoles($faker, 400);
$count = setRoles($faker, 400);
Console::info("\n{$count} roles:");
$report[] = [
'roles' => $count,
'results' => runQueries($database, $limit)
];

$count = addRoles($faker, 500);
$count = setRoles($faker, 500);
Console::info("\n{$count} roles:");
$report[] = [
'roles' => $count,
'results' => runQueries($database, $limit)
];

$count = addRoles($faker, 1000);
$count = setRoles($faker, 1000);
Console::info("\n{$count} roles:");
$report[] = [
'roles' => $count,
'results' => runQueries($database, $limit)
];

if (!file_exists('bin/view/results')) {
mkdir('bin/view/results', 0777, true);
\mkdir('bin/view/results', 0777, true);
}

$time = time();
$f = fopen("bin/view/results/{$adapter}_{$name}_{$limit}_{$time}.json", 'w');
fwrite($f, json_encode($report));
fclose($f);
$time = \time();
$results = \fopen("bin/view/results/{$adapter}_{$name}_{$limit}_{$time}.json", 'w');
\fwrite($results, \json_encode($report));
\fclose($results);
});


$cli
->error()
->inject('error')
->action(function (Exception $error) {
Console::error($error->getMessage());
});
->error()
->inject('error')
->action(function (Exception $error) {
Console::error($error->getMessage());
});

function setRoles($faker, $count): int
{
for ($i = 0; $i < $count; $i++) {
Authorization::setRole($faker->numerify('user####'));
}
return \count(Authorization::getRoles());
}

function runQueries(Database $database, int $limit)
function runQueries(Database $database, int $limit): array
{
$results = [];
// Recent travel blogs

$results[] = runQuery([
// Recent travel blogs
$results["Querying greater than, equal[1] and limit"] = runQuery([
Query::greaterThan('created', '2010-01-01 05:00:00'),
Query::equal('genre', ['travel']),
Query::limit($limit)
], $database);

// Favorite genres

$results[] = runQuery([
$results["Querying equal[3] and limit"] = runQuery([
Query::equal('genre', ['fashion', 'finance', 'sports']),
Query::limit($limit)
], $database);

// Popular posts

$results[] = runQuery([
$results["Querying greaterThan, limit({$limit})"] = runQuery([
Query::greaterThan('views', 100000),
Query::limit($limit)
], $database);

// Fulltext search

$results[] = runQuery([
$results["Query search, limit({$limit})"] = runQuery([
Query::search('text', 'Alice'),
Query::limit($limit)
], $database);

return $results;
}
// Tags contain query
$results["Querying contains[1], limit({$limit})"] = runQuery([
Query::contains('tags', ['tag1']),
Query::limit($limit)
], $database);

function addRoles($faker, $count)
{
for ($i = 0; $i < $count; $i++) {
Authorization::setRole($faker->numerify('user####'));
}
return count(Authorization::getRoles());
return $results;
}

function runQuery(array $query, Database $database)
{
$info = array_map(function ($q) {
/** @var $q Query */
return $q->getAttribute() . ' : ' . $q->getMethod() . ' : ' . implode(',', $q->getValues());
$info = array_map(function (Query $q) {
return $q->getAttribute() . ': ' . $q->getMethod() . ' = ' . implode(',', $q->getValues());
}, $query);

Console::log('Running query: [' . implode(', ', $info) . ']');
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"roles":2,"results":[0.004142045974731445,0.0007698535919189453,0.0006570816040039062,4.3037660121917725]},{"roles":102,"results":[0.0031499862670898438,0.0012857913970947266,0.0013210773468017578,4.130218029022217]},{"roles":491,"results":[0.4965331554412842,0.36292195320129395,0.30788612365722656,4.9501330852508545]},{"roles":964,"results":[0.44646310806274414,0.44009995460510254,0.37430691719055176,4.239892959594727]},{"roles":1829,"results":[0.6837189197540283,1.6691820621490479,1.3487520217895508,98.95817399024963]}]
Loading
Loading