Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion presto-docs/src/main/sphinx/develop/presto-native.rst
Original file line number Diff line number Diff line change
Expand Up @@ -126,4 +126,36 @@ There is also an additional parameter:
If the time period is less than or equal to the parameter value, the request
is valid.
If the time period exceeds the parameter value, the request is rejected as
authentication failure (HTTP 401).
authentication failure (HTTP 401).

Async Data Cache and Prefetching
--------------------------------

``num-connector-io-threads``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^

* **Type** ``integer``
* **Default value:** ``30``
* **Presto on Spark default value:** ``0``

Size of IO executor for connectors to do preload/prefetch. Prefetch is
disabled if ``num-connector-io-threads`` is set to zero.

``async-data-cache-enabled``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^

* **Type** ``bool``
* **Default value:** ``true``
* **Presto on Spark default value:** ``false``

Whether async data cache is enabled. Setting ``async-data-cache-enabled``
to ``false`` disables split prefetching in table scan.

``async-cache-ssd-gb``
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

consistency: async-data-cache-enabled vs. async-cache-ssd-gb - one says data-cache, the other just cache. Would be nice to fix.

Copy link
Contributor Author

@Yuhta Yuhta Oct 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That will be a big risky change, rather not changing it here

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Understood. Perhaps, file a GitHub issue about this.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

^^^^^^^^^^^^^^^^^^^^^^

* **Type** ``integer``
* **Default value:** ``0``
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we missing some other configs re: SSD. I assume one needs to specify rott directory or something, no?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can use async data cache without SSD

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it, but if I want to use SSD I assume I need more properties than just size, no?

Copy link
Contributor Author

@Yuhta Yuhta Oct 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes you will need to set SSD size in another property to indicate how much space is available


Size of the SSD cache when async data cache is enabled. Must be zero if
``async-data-cache-enabled`` is ``false``.
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ public class NativeExecutionSystemConfig
// TODO: others use "-" separator and this property use _ separator. Fix them.
private static final String HTTP_EXEC_THREADS = "http_exec_threads";
private static final String NUM_IO_THREADS = "num-io-threads";
private static final String ASYNC_DATA_CACHE_ENABLED = "async-data-cache-enabled";
private static final String ASYNC_CACHE_SSD_GB = "async-cache-ssd-gb";
private static final String NUM_CONNECTOR_IO_THREADS = "num-connector-io-threads";
private static final String PRESTO_VERSION = "presto.version";
private static final String SHUTDOWN_ONSET_SEC = "shutdown-onset-sec";
// Memory related configurations.
Expand Down Expand Up @@ -104,6 +107,9 @@ public class NativeExecutionSystemConfig
private String httpsCertPath = "";
private String httpsKeyPath = "";
private int numIoThreads = 30;
private boolean asyncDataCacheEnabled; // false
private int asyncCacheSsdGb; // 0
private int numConnectorIoThreads; // 0
private int shutdownOnsetSec = 10;
private int systemMemoryGb = 10;
// Reserve 2GB from system memory for system operations such as disk
Expand Down Expand Up @@ -139,6 +145,9 @@ public Map<String, String> getAllProperties()
.put(HTTPS_KEY_PATH, String.valueOf(getHttpsKeyPath()))
.put(HTTP_EXEC_THREADS, String.valueOf(getHttpExecThreads()))
.put(NUM_IO_THREADS, String.valueOf(getNumIoThreads()))
.put(ASYNC_DATA_CACHE_ENABLED, String.valueOf(getAsyncDataCacheEnabled()))
.put(ASYNC_CACHE_SSD_GB, String.valueOf(getAsyncCacheSsdGb()))
.put(NUM_CONNECTOR_IO_THREADS, String.valueOf(getNumConnectorIoThreads()))
.put(PRESTO_VERSION, getPrestoVersion())
.put(SHUTDOWN_ONSET_SEC, String.valueOf(getShutdownOnsetSec()))
.put(SYSTEM_MEMORY_GB, String.valueOf(getSystemMemoryGb()))
Expand Down Expand Up @@ -323,6 +332,42 @@ public int getNumIoThreads()
return numIoThreads;
}

@Config(ASYNC_DATA_CACHE_ENABLED)
public NativeExecutionSystemConfig setAsyncDataCacheEnabled(boolean asyncDataCacheEnabled)
{
this.asyncDataCacheEnabled = asyncDataCacheEnabled;
return this;
}

public boolean getAsyncDataCacheEnabled()
{
return asyncDataCacheEnabled;
}

@Config(ASYNC_CACHE_SSD_GB)
public NativeExecutionSystemConfig setAsyncCacheSsdGb(int asyncCacheSsdGb)
{
this.asyncCacheSsdGb = asyncCacheSsdGb;
return this;
}

public int getAsyncCacheSsdGb()
{
return asyncCacheSsdGb;
}

@Config(NUM_CONNECTOR_IO_THREADS)
public NativeExecutionSystemConfig setNumConnectorIoThreads(int numConnectorIoThreads)
{
this.numConnectorIoThreads = numConnectorIoThreads;
return this;
}

public int getNumConnectorIoThreads()
{
return numConnectorIoThreads;
}

@Config(SHUTDOWN_ONSET_SEC)
public NativeExecutionSystemConfig setShutdownOnsetSec(int shutdownOnsetSec)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ public void testNativeExecutionSystemConfig()
.setHttpsCertPath("")
.setHttpsKeyPath("")
.setNumIoThreads(30)
.setAsyncDataCacheEnabled(false)
.setAsyncCacheSsdGb(0)
.setNumConnectorIoThreads(0)
.setShutdownOnsetSec(10)
.setSystemMemoryGb(10)
.setQueryMemoryGb(new DataSize(8, DataSize.Unit.GIGABYTE))
Expand Down Expand Up @@ -103,6 +106,9 @@ public void testNativeExecutionSystemConfig()
.setHttpsCertPath("/tmp/non_existent.cert")
.setHttpsKeyPath("/tmp/non_existent.key")
.setNumIoThreads(50)
.setAsyncDataCacheEnabled(true)
.setAsyncCacheSsdGb(1000)
.setNumConnectorIoThreads(30)
.setPrestoVersion("presto-version")
.setShutdownOnsetSec(30)
.setSystemMemoryGb(40)
Expand Down