Skip to content
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
6f22fa6
modern python
chayim May 16, 2021
2e4bca1
adding nighties
chayim May 16, 2021
6cf5bc5
adding docker file skeleton that builds
chayim May 16, 2021
ef3d224
linter cleanup
chayim May 16, 2021
34c46b5
docs update
chayim May 18, 2021
e5893e2
sample docker, linters, tox updates
chayim May 18, 2021
3ca1c7e
snyk added to the README, poetry pypi workflow
chayim May 18, 2021
1161cb7
add support
AvitalFineRedis May 18, 2021
aa8cff9
add tests
AvitalFineRedis May 18, 2021
044fa97
tox, with poetry
chayim May 18, 2021
24e0fef
ai, from bloom
chayim May 19, 2021
d71fe62
latest
chayim May 19, 2021
b9b31e3
linter comments
chayim May 19, 2021
fbc5cda
lifting dockers
chayim May 19, 2021
db860f6
unifying the pypi settings
chayim May 19, 2021
a313a28
merging master
chayim May 19, 2021
5bf1426
linters
chayim May 19, 2021
d798fb7
Merge branch 'scriptexecute_support' of https://github.com/RedisAI/re…
AvitalFineRedis May 20, 2021
0056bc3
reset
AvitalFineRedis May 20, 2021
d17b278
Some PR fixes
AvitalFineRedis May 20, 2021
cabbdc3
new files
AvitalFineRedis May 20, 2021
1f5011b
Update test-requirements.txt
AvitalFineRedis May 20, 2021
4a2b3a0
Update postprocessor.py
AvitalFineRedis May 20, 2021
a2e4b65
Update setup.py
AvitalFineRedis May 20, 2021
24537c8
Update test.py
AvitalFineRedis May 23, 2021
53b1106
Update client.py
AvitalFineRedis May 25, 2021
2a60e57
add scriptstore, and update scriptexecute & tests
AvitalFineRedis Jul 7, 2021
6ea2a9f
fix examples
AvitalFineRedis Jul 7, 2021
569604e
typo
AvitalFineRedis Jul 7, 2021
918766e
merge again
AvitalFineRedis Jul 7, 2021
7866203
flake8 warnings
AvitalFineRedis Jul 7, 2021
300db9d
flake8 warnings
AvitalFineRedis Jul 7, 2021
b59152e
Merge remote-tracking branch 'origin/scriptexecute_support' into scri…
AvitalFineRedis Jul 7, 2021
12ad6bd
typo
AvitalFineRedis Jul 7, 2021
9a6344a
fix PR comments
AvitalFineRedis Jul 18, 2021
b8e77b9
fix PR comments
AvitalFineRedis Jul 18, 2021
964d8ba
fix imports
AvitalFineRedis Jul 18, 2021
f17dab0
fix imports
AvitalFineRedis Jul 18, 2021
c14248b
Merge remote-tracking branch 'origin/scriptexecute_support' into scri…
AvitalFineRedis Jul 18, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion redisai/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
from .client import Client
from .client import Client # noqa

__version__ = "1.0.2"
66 changes: 61 additions & 5 deletions redisai/client.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import warnings
from functools import partial, wraps
from typing import Any, AnyStr, ByteString, List, Sequence, Union
from functools import wraps
from typing import AnyStr, ByteString, List, Sequence, Union

import numpy as np
from deprecated import deprecated
Expand Down Expand Up @@ -65,8 +65,8 @@ def pipeline(self, transaction: bool = True, shard_hint: bool = None) -> "Pipeli
self.enable_postprocess,
self.connection_pool,
self.response_callbacks,
transaction=True,
shard_hint=None,
transaction=transaction,
shard_hint=shard_hint,
)

def dag(
Expand Down Expand Up @@ -622,6 +622,7 @@ def scriptdel(self, key: AnyStr) -> str:
res = self.execute_command(*args)
return res if not self.enable_postprocess else processor.scriptdel(res)

@deprecated(version="1.2.0", reason="Use scriptexecute instead")
def scriptrun(
self,
key: AnyStr,
Expand All @@ -642,7 +643,7 @@ def scriptrun(
Tensor(s) which is already saved in the RedisAI using a tensorset call. These
tensors will be used as the input for the modelrun
outputs : Union[AnyStr, List[AnyStr]]
keys on which the outputs to be saved. If those keys exist already, modelrun
keys on which the outputs to be saved. If those keys exist already, scriptrun
will overwrite them with new values

Returns
Expand All @@ -659,6 +660,61 @@ def scriptrun(
res = self.execute_command(*args)
return res if not self.enable_postprocess else processor.scriptrun(res)

def scriptexecute(
self,
key: AnyStr,
function: str,
keys: Union[AnyStr, Sequence[AnyStr]],
inputs: Union[AnyStr, Sequence[Union[AnyStr, Sequence[AnyStr]]]] = None,
outputs: Union[AnyStr, Sequence[AnyStr]] = None,
timeout: int = None,
) -> str:
"""
Run an already set script. Similar to modelexecute

Parameters
----------
key : AnyStr
Script key
function : str
Name of the function in the ``script``
keys : Union[AnyStr, Sequence[AnyStr]]
Either a squence of key names that the script will access before, during and
Comment thread
AvitalFineRedis marked this conversation as resolved.
Outdated
after its execution, or a tag which all those keys share.
inputs : Union[AnyStr, Sequence[Union[AnyStr, Sequence[AnyStr]]]]
The inputs can be tensor key name, string , int or float.
These inputs will be used as the input for the scriptexecute function.
The order of the input should be aligned with the order of their respected
parameter at the function signature.
When one of the elements in the inputs list is a list (or a tuple), that element
will be treated as LIST_INPUTS in the command executaion.
outputs : Union[AnyStr, List[AnyStr]]
keys on which the outputs to be saved. If those keys exist already, scriptexecute
will overwrite them with new values.
timeout : int
The max number on milisecinds that may pass before the request is prossced
(meaning that the result will not be computed after that time and TIMEDOUT
is returned in that case).

Returns
-------
str
'OK' if success, raise an exception otherwise

Example
-------
>>> con.scriptexecute('ket', 'bar', keys=['a', 'b', 'c'], inputs=['a', 'b'], outputs=['c'])
Comment thread
AvitalFineRedis marked this conversation as resolved.
Outdated
'OK'
>>> con.scriptexecute('myscript{tag}', 'addn',
>>> keys=['{tag}'],
Comment thread
AvitalFineRedis marked this conversation as resolved.
Outdated
>>> inputs=['mytensor1{tag}', ['mytensor2{tag}', 'mytensor3{tag}']],
>>> outputs=['result{tag}'])
'OK'
"""
Comment thread
AvitalFineRedis marked this conversation as resolved.
args = builder.scriptexecute(key, function, keys, inputs, outputs, timeout)
res = self.execute_command(*args)
return res if not self.enable_postprocess else processor.scriptexecute(res)

def scriptscan(self) -> List[List[AnyStr]]:
"""
Returns the list of all the script in the RedisAI server. Scriptscan API is
Expand Down
48 changes: 45 additions & 3 deletions redisai/command_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def modelstore(
"Inputs and outputs keywords should not be specified for this backend"
)
chunk_size = 500 * 1024 * 1024 # TODO: this should be configurable.
data_chunks = [data[i : i + chunk_size] for i in range(0, len(data), chunk_size)]
data_chunks = [data[i: i + chunk_size] for i in range(0, len(data), chunk_size)]
# TODO: need a test case for this
args += ["BLOB", *data_chunks]
return args
Expand Down Expand Up @@ -102,7 +102,7 @@ def modelset(
args += ["INPUTS", *utils.listify(inputs)]
args += ["OUTPUTS", *utils.listify(outputs)]
chunk_size = 500 * 1024 * 1024
data_chunks = [data[i : i + chunk_size] for i in range(0, len(data), chunk_size)]
data_chunks = [data[i: i + chunk_size] for i in range(0, len(data), chunk_size)]
# TODO: need a test case for this
args += ["BLOB", *data_chunks]
return args
Expand Down Expand Up @@ -228,10 +228,12 @@ def scriptdel(name: AnyStr) -> Sequence:

def scriptrun(
name: AnyStr,
function: AnyStr,
function: str,
inputs: Union[AnyStr, Sequence[AnyStr]],
outputs: Union[AnyStr, Sequence[AnyStr]],
) -> Sequence:
if name is None or function is None:
raise ValueError("Missing required arguments for script run command")
args = (
"AI.SCRIPTRUN",
name,
Expand All @@ -243,6 +245,46 @@ def scriptrun(
)
return args

def scriptexecute(
name: AnyStr,
function: str,
keys: Union[AnyStr, Sequence[AnyStr]],
inputs: Union[AnyStr, Sequence[Union[AnyStr, Sequence[AnyStr]]]],
outputs: Union[AnyStr, Sequence[AnyStr]],
timeout: int,
) -> Sequence:
if name is None or function is None or keys is None:
raise ValueError("Missing required arguments for script execute command")
args = [
Comment thread
AvitalFineRedis marked this conversation as resolved.
Outdated
"AI.SCRIPTEXECUTE",
name,
function,
"KEYS",
Comment thread
AvitalFineRedis marked this conversation as resolved.
Outdated
len(utils.listify(keys)),
*utils.listify(keys),
]

if inputs is not None:
Comment thread
AvitalFineRedis marked this conversation as resolved.
temp_inputs = []
if not isinstance(inputs, (list, tuple)):
args += ["INPUTS", 1, inputs]
else:
for elem in inputs:
if isinstance(elem, (list, tuple)):
if temp_inputs:
args += ["INPUTS", len(temp_inputs), *temp_inputs]
temp_inputs = []
args += ["LIST_INPUTS", len(utils.listify(elem)), *utils.listify(elem)]
else:
temp_inputs.append(elem)
if temp_inputs:
args += ["INPUTS", len(temp_inputs), *temp_inputs]
if outputs is not None:
args += ["OUTPUTS", len(utils.listify(outputs)), *utils.listify(outputs)]
if timeout is not None:
args += ["TIMEOUT", timeout]

return args

def scriptscan() -> Sequence:
return ("AI._SCRIPTSCAN",)
Expand Down
5 changes: 3 additions & 2 deletions redisai/dag.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
from functools import partial
from typing import Any, AnyStr, List, Sequence, Union
from typing import AnyStr, Union, Sequence, Any, List

import numpy as np

from redisai import command_builder as builder
from redisai.postprocessor import Processor
from redisai import command_builder as builder

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why the changes here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it's mine :0
I saw here some of Chayim's commits

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as above :)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see here some functions that use builder. The DagTestCase does not pass without this import


processor = Processor()

Expand Down
5 changes: 3 additions & 2 deletions redisai/pipeline.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
import warnings
from functools import partial
from typing import AnyStr, Sequence, Union
from typing import AnyStr, Union, Sequence
Comment thread
AvitalFineRedis marked this conversation as resolved.
Outdated

import numpy as np
import redis

from redisai import command_builder as builder
import redis

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why there is a diff here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PipelineTest does not pass without it

from redisai.postprocessor import Processor


processor = Processor()


Expand Down
3 changes: 1 addition & 2 deletions redisai/postprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,14 @@ def infoget(res):
decoder = staticmethod(decoder)
decoding_functions = (
"loadbackend",
"modelstore",
"modelset",
"modeldel",
"modelexecute",
"modelrun",
"tensorset",
"scriptset",
"scriptdel",
"scriptrun",
"scriptexecute",
"inforeset",
)
for fn in decoding_functions:
Expand Down
4 changes: 2 additions & 2 deletions redisai/utils.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from typing import AnyStr, ByteString, Callable, List, Sequence, Union

from typing import Union, ByteString, Sequence, List, AnyStr, Callable
import numpy as np


dtype_dict = {
"float": "FLOAT",
"double": "DOUBLE",
Expand Down
32 changes: 0 additions & 32 deletions setup.py

This file was deleted.

9 changes: 0 additions & 9 deletions test-requirements.txt

This file was deleted.

Loading