Executor
+types.h
+-
+
-
+template<>
struct TypeTraits<std::int8_t>
+ - +
-
+
-
+template<>
struct TypeTraits<std::int32_t>
+ - +
-
+
-
+template<>
struct TypeTraits<std::int64_t>
+ - +
-
+
-
+template<>
struct TypeTraits<std::uint8_t>
+ - +
-
+
-
+namespace tensorrt_llm
-
+
-
+namespace executor
- +
Typedefs
+ + +-
+
-
+using SizeType32 = std::int32_t
-
+
-
+using FloatType = float
-
+
-
+using TokenIdType = std::int32_t
-
+
-
+using VecTokens = std::vector<TokenIdType>
-
+
-
+using IdType = std::uint64_t
-
+
-
+using IterationType = std::uint64_t
-
+
-
+using RandomSeedType = std::uint64_t
-
+
-
+using StreamPtr = std::shared_ptr<tensorrt_llm::runtime::CudaStream>
-
+
-
+using MillisecondsType = std::chrono::milliseconds
-
+
-
+using LogitsPostProcessor = std::function<void(IdType, Tensor&, BeamTokens const&, StreamPtr const&, std::optional<IdType>)>
-
+
-
+using LogitsPostProcessorMap = std::unordered_map<std::string, LogitsPostProcessor>
+ -
+
-
+using LogitsPostProcessorBatched = std::function<void(std::vector<IdType> const&, std::vector<Tensor>&, std::vector<std::reference_wrapper<BeamTokens const>> const&, StreamPtr const&, std::vector<std::optional<IdType>> const&)>
-- --- -- Executor
-- +types.h
--
-
-
-template<>
struct TypeTraits<std::int8_t>
- - -+
Public Static Attributes
--
-
-
-static constexpr auto value = DataType::kINT8
-
+using MedusaChoices = std::vector<std::vector<SizeType32>>
-
+using EagleChoices = std::vector<std::vector<SizeType32>>
-
+using PriorityType = float
-
+using BufferView = std::basic_string_view<uint8_t>
-
+
-
+
-
+
-
+
+Enums
+-
+
-
+enum class DataType
Values:
+-
+
-
+enumerator kBOOL
-
+
-
+enumerator kUINT8
-
+
-
+enumerator kINT8
-
+
-
+enumerator kINT32
-
+
-
+enumerator kINT64
-
+
-
+enumerator kBF16
-
+
-
+enumerator kFP8
-
+
-
+enumerator kFP16
-
+
-
+enumerator kFP32
-
+
-
+enumerator kUNKNOWN
-
+enumerator kBOOL
-
-
-
-template<>
struct TypeTraits<std::int32_t>
-
-
-
-
-template<>
struct TypeTraits<std::int64_t>
- - -+
Public Static Attributes
--
+
-
+enumerator kCPU_PINNED
-
+
-
+enumerator kCPU_PINNEDPOOL
-
+
-
+enumerator kGPU
-
+
-
+enumerator kUVM
-
+
-
+enumerator kUNKNOWN
-
+enumerator kCPU_PINNED
-
+
-
+enum class ModelType
Values:
+-
+
-
+enumerator kDECODER_ONLY
-
+
-
+enumerator kENCODER_ONLY
-
+
-
+enumerator kENCODER_DECODER
-
+enumerator kDECODER_ONLY
-
+
-
+enum class BatchingType
The batching type.
+Values:
+-
+
-
+enumerator kSTATIC
STATIC refers to the traditional batching scheme with a batch of requests running in lockstep until the full generation for all of them is complete. Requests in a batch are all padded up to the maximum input and output sequence length of any member of the batch.
+
-
+
-
+enumerator kINFLIGHT
INFLIGHT refers to a scheme where newly arrived requests are dynamically incorporated into the batch under execution, and requests are returned as soon as the end condition is met without any padding.
+
-
+enumerator kSTATIC
-
+
-
+enum class CapacitySchedulerPolicy
The policy used to select the subset of available requests in each iteration of the executor generation loop.
+Values:
+-
+
-
+enumerator kMAX_UTILIZATION
MAX_UTILIZATION packs as many requests as the underlying TRT engine can support in any iteration of the InflightBatching generation loop. While this is expected to maximize GPU throughput, it might require that some requests be paused and restarted depending on peak KV cache memory availability.
+
-
+
-
+enumerator kGUARANTEED_NO_EVICT
GUARANTEED_NO_EVICT uses KV cache more conservatively guaranteeing that a request, once started, will run to completion without eviction.
+
-
+
-
+enumerator kSTATIC_BATCH
kSTATIC_BATCH does not schedule new requests until all requests in current batch are completed. Similar to kGUARANTEED_NO_EVICT, requests will run to completion without eviction.
+
-
+enumerator kMAX_UTILIZATION
-
+
-
+enum class ContextChunkingPolicy
Values:
+-
+
-
+enumerator kFIRST_COME_FIRST_SERVED
Sequential chunking, complete the unfinished context phase first.
-
-
-
-template<>
struct TypeTraits<std::uint8_t>
-
-
-
-
-namespace tensorrt_llm
-
-
-
-namespace executor
- -+
Typedefs
--
-
-
-using TensorPtr = std::shared_ptr<Tensor>
-
+enum class CommunicationType
Values:
+-
+
-
+enumerator kMPI
-
-
-
-using SizeType32 = std::int32_t
-
+enumerator kMPI
-
-using FloatType = float
-
+enum class CommunicationMode
Values:
+-
+
-
+enumerator kLEADER
-
-
-
-using TokenIdType = std::int32_t
-
+enumerator kORCHESTRATOR
-
-using VecTokens = std::vector<TokenIdType>
-
+
-
-
-
+enumerator kLEADER
-
+enum class RequestStage
Enum class that represents the state of a request.
+Values:
+-
+
-
+enumerator kQUEUED
Request that have been received but not yet included in the active requests (due to constraints such as maximum batch size for example).
+
-
-
-
-using IdType = std::uint64_t
-
+
-
+enumerator kENCODER_IN_PROGRESS
Active request in encoder phase.
+
-
+
-
+enumerator kCONTEXT_IN_PROGRESS
Active request in context phase.
+
-
-
-
-using IterationType = std::uint64_t
-
+
-
+enumerator kGENERATION_IN_PROGRESS
Active request in generation phase.
+
-
-
-
-using RandomSeedType = std::uint64_t
-
+
-
+enumerator kGENERATION_COMPLETE
Active request for which generation has completed.
+
-
+enumerator kQUEUED
-
-using StreamPtr = std::shared_ptr<tensorrt_llm::runtime::CudaStream>
-
+enum class FinishReason
The reason why the model stopped generating tokens for a request.
+Values:
+-
+
-
+enumerator kNOT_FINISHED
The request is not finished.
+
-
-
-
-using MillisecondsType = std::chrono::milliseconds
-
+
-
+enumerator kEND_ID
The request finished because the end id was generated.
+
-
-
-
-using LogitsPostProcessor = std::function<void(IdType, Tensor&, BeamTokens const&, StreamPtr const&, std::optional<IdType>)>
-
+
-
+enumerator kSTOP_WORDS
The request finished because a stop word was generated.
+
-
-
-
-using LogitsPostProcessorMap = std::unordered_map<std::string, LogitsPostProcessor>
-
+
-
+enumerator kLENGTH
The request finished because the maximum number of tokens was reached.
+
-
-
-
-using LogitsPostProcessorBatched = std::function<void(std::vector<IdType> const&, std::vector<Tensor>&, std::vector<std::reference_wrapper<BeamTokens const>> const&, StreamPtr const&, std::vector<std::optional<IdType>> const&)>
-
+
-
+enumerator kTIMED_OUT
The request finished because it got timed out (via the mAllotedTime parameter)
+
-
-
-
-using MedusaChoices = std::vector<std::vector<SizeType32>>
-
+
-
+enumerator kCANCELLED
The request was cancelled by calling cancelRequest.
+
-
-
-
-using EagleChoices = std::vector<std::vector<SizeType32>>
-
+enumerator kNOT_FINISHED
-
-using PriorityType = float
-
+
-
-
-
+
-
+
-
-
-
+
-
-
++Functions
+-
+
-
+std::ostream &operator<<(std::ostream &os, CapacitySchedulerPolicy policy)
-
-
-
-using BufferView = std::basic_string_view<uint8_t>
-
+std::ostream &operator<<(std::ostream &os, ContextChunkingPolicy policy)
-
+
-
+
-
+struct DebugTensorsPerIteration
- +#include <types.h>+
Struct that holds the debug tensors in an iteration.
+
-
-
-
-enumerator kINT32
-
+class DecodingMode
- +#include <types.h>+
mode of the decoder
+ +++Public Functions
+-
+
-
+inline auto constexpr useTemperature(bool useTemp)
-
-
-
-enumerator kBF16
-
+inline auto constexpr useOccurrencePenalties(bool usePenalty)
-
-enumerator kFP8
-
+inline auto constexpr usePresencePenalty(bool usePenalty)
-
-enumerator kFP16
-
+inline auto constexpr useRepetitionPenalty(bool usePenalty)
-
-enumerator kFP32
-
+inline auto constexpr useFrequencyPenalty(bool usePenalty)
-
-enum class RequestType
Values:
--
+inline auto constexpr useStopWords(bool stopWords)
-
-enum class MemoryType
Values:
--
+inline bool constexpr isTopKandTopP() const
-
-enum class ModelType
Values:
--
+inline bool constexpr isExplicitDraftTokens() const
-
-enum class BatchingType
The batching type.
-Values:
--
-
-
-enumerator kSTATIC
STATIC refers to the traditional batching scheme with a batch of requests running in lockstep until the full generation for all of them is complete. Requests in a batch are all padded up to the maximum input and output sequence length of any member of the batch.
-
-
+
-
+inline bool constexpr isExternalDraftTokens() const
-
-
-
-enumerator kINFLIGHT
INFLIGHT refers to a scheme where newly arrived requests are dynamically incorporated into the batch under execution, and requests are returned as soon as the end condition is met without any padding.
-
-
+
-
+inline bool constexpr isEagle() const
-
-enumerator kSTATIC
-
+inline bool constexpr isUseTemperature() const
-
-enum class CapacitySchedulerPolicy
The policy used to select the subset of available requests in each iteration of the executor generation loop.
-Values:
--
-
-
-enumerator kMAX_UTILIZATION
MAX_UTILIZATION packs as many requests as the underlying TRT engine can support in any iteration of the InflightBatching generation loop. While this is expected to maximize GPU throughput, it might require that some requests be paused and restarted depending on peak KV cache memory availability.
-
-
+
-
+inline bool constexpr isUsePresencePenalty() const
-
-
-
-enumerator kGUARANTEED_NO_EVICT
GUARANTEED_NO_EVICT uses KV cache more conservatively guaranteeing that a request, once started, will run to completion without eviction.
-
-
+
-
+inline bool constexpr isUseFrequencyPenalty() const
-
-
-
-enumerator kSTATIC_BATCH
kSTATIC_BATCH does not schedule new requests until all requests in current batch are completed. Similar to kGUARANTEED_NO_EVICT, requests will run to completion without eviction.
-
-
+
-
+inline bool constexpr isUseRepetitionPenalty() const
-
-enumerator kMAX_UTILIZATION
-
+inline bool constexpr isUseMinLength() const
-
-enum class ContextChunkingPolicy
Values:
--
-
-
-enumerator kFIRST_COME_FIRST_SERVED
Sequential chunking, complete the unfinished context phase first.
-
-
+
-
+inline bool constexpr isUseOccurrencePenalty() const
-
-
-
-enumerator kEQUAL_PROGRESS
Iterate through each context request in sequence and attempt to increase its chunk count until the constraint is exceeded.
-
-
+
-
+inline bool constexpr isUsePenalty() const
-
-enumerator kFIRST_COME_FIRST_SERVED
-
+inline bool constexpr isUseBanWords() const
-
-enum class CommunicationType
Values:
--
+inline bool constexpr isUseBanTokens() const
-
-enum class CommunicationMode
Values:
--
+inline bool constexpr isUseExplicitEosStop() const
-
-enum class RequestStage
Enum class that represents the state of a request.
-Values:
--
-
-
-enumerator kQUEUED
Request that have been received but not yet included in the active requests (due to constraints such as maximum batch size for example).
-
-
+
-
+inline bool constexpr isUseStopCriteria() const
-
-
-
-enumerator kENCODER_IN_PROGRESS
Active request in encoder phase.
-
-
+
-
+inline bool operator==(DecodingMode const &other) const
-
-
-
-enumerator kCONTEXT_IN_PROGRESS
Active request in context phase.
-
-
+
-
+inline explicit constexpr DecodingMode(UnderlyingType state)
-
-
-
-enumerator kGENERATION_IN_PROGRESS
Active request in generation phase.
-
-
+
-
+inline constexpr UnderlyingType getState() const
-
-
-
-enumerator kGENERATION_COMPLETE
Active request for which generation has completed.
+
-
-enumerator kQUEUED
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
+Public Static Functions
+-
+
-
+static inline auto constexpr Auto()
No mode specified. Config will be determined from the beam width of the first request at runtime TopKTopP if beamWidth == 1, BeamSearch otherwise.
-
+
-
+static inline auto constexpr TopK()
-
-
-
-enum class FinishReason
The reason why the model stopped generating tokens for a request.
-Values:
--
-
-
-enumerator kNOT_FINISHED
The request is not finished.
-
-
+
-
+static inline auto constexpr TopP()
-
-
-
-enumerator kEND_ID
The request finished because the end id was generated.
-
-
+
-
+static inline auto constexpr TopKTopP()
-
-
-
-enumerator kSTOP_WORDS
The request finished because a stop word was generated.
-
-
+
-
+static inline auto constexpr BeamSearch()
-
-
-
-enumerator kLENGTH
The request finished because the maximum number of tokens was reached.
-
-
+
-
+static inline auto constexpr Medusa()
-
-
-
-enumerator kTIMED_OUT
The request finished because it got timed out (via the mAllotedTime parameter)
-
-
+
-
+static inline auto constexpr Lookahead()
-
-
-
-enumerator kCANCELLED
The request was cancelled by calling cancelRequest.
-
-
+
-
+static inline auto constexpr ExplicitDraftTokens()
-
-enumerator kNOT_FINISHED
-
+
-
+static inline auto constexpr ExternalDraftTokens()
-
+
-
+static inline auto constexpr Eagle()
--Functions
+Private Functions
-
-
-
-std::ostream &operator<<(std::ostream &os, CapacitySchedulerPolicy policy)
-
+inline bool constexpr anyBitSet(UnderlyingType bits) const
-
-
-
-std::ostream &operator<<(std::ostream &os, ContextChunkingPolicy policy)
-
+inline bool constexpr allBitSet(UnderlyingType bits) const
-
-
-
-template<typename T, bool = false>
struct TypeTraits
- - -#include <types.h>-
For converting a C++ data type to a
-TrtLmmDataType.
-
+
-
+inline UnderlyingType constexpr setBitTo(UnderlyingType state, bool x)
-
-
-
-template<>
struct TypeTraits<float>
- - -+
Public Static Attributes
++-Private Members
-
-
-
-static constexpr auto value = DataType::kFP32
-
+UnderlyingType mState = {}
-
-static constexpr auto value = DataType::kFP32
++Private Static Attributes
+-
+
-
+static UnderlyingType constexpr kUseRepetitionPenalties = {1u << 0}
-
-
-
-template<>
struct TypeTraits<half>
- - --
Public Static Attributes
-
-
-
-static constexpr auto value = DataType::kFP16
-
+static UnderlyingType constexpr kUseFrequencyPenalties = {1u << 1}
-
-static constexpr auto value = DataType::kFP16
-
+
-
+static UnderlyingType constexpr kUsePresencePenalties = {1u << 2}
-
-
- -template<> int8_t > -
- --
Public Static Attributes
-
-
-
-static constexpr auto value = DataType::kINT8
-
+static UnderlyingType constexpr kUseTemperature = {1u << 3}
-
-static constexpr auto value = DataType::kINT8
-
+
-
+static UnderlyingType constexpr kUseMinLength = {1u << 4}
-
-
- -template<> int32_t > -
- --
Public Static Attributes
-
-
-
-static constexpr auto value = DataType::kINT32
-
+static UnderlyingType constexpr kUseBanWords = {1u << 5}
-
-static constexpr auto value = DataType::kINT32
-
+
-
+static UnderlyingType constexpr kUseStopWords = {1u << 6}
-
-
- -template<> int64_t > -
- --
Public Static Attributes
-
-
-
-static constexpr auto value = DataType::kINT64
-
+static UnderlyingType constexpr kUseMaxLengthStop = {1u << 7}
-
-static constexpr auto value = DataType::kINT64
-
+
-
+static UnderlyingType constexpr kUseExplicitEosStop = {1u << 8}
-
-
-
-template<>
struct TypeTraits<bool>
- - --
Public Static Attributes
-
-
-
-static constexpr auto value = DataType::kBOOL
-
+static UnderlyingType constexpr kUseNoRepeatNgramSize = {1u << 9}
-
-static constexpr auto value = DataType::kBOOL
-
+
-
+static UnderlyingType constexpr kStandardStopCriteria = {kUseStopWords | kUseMaxLengthStop}
-
-
- -template<> uint8_t > -
- --
Public Static Attributes
-
-
-
-static constexpr auto value = DataType::kUINT8
-
+static UnderlyingType constexpr kUseOccurrencePenalties{kUseRepetitionPenalties | kUseFrequencyPenalties | kUsePresencePenalties}
-
-static constexpr auto value = DataType::kUINT8
-
+
-
+static UnderlyingType constexpr kUsePenalties = {kUseOccurrencePenalties | kUseTemperature | kUseMinLength}
-
-
-
-template<typename T>
struct TypeTraits<T*>
- - --
Public Static Attributes
-
-
-
-static constexpr auto value = DataType::kINT64
-
+static UnderlyingType constexpr kUseBanTokens = {kUseNoRepeatNgramSize | kUseBanWords}
-
-static constexpr auto value = DataType::kINT64
-
+
-
+static SizeType32 constexpr kNumFlags = {10}
-
-
-
-struct KvCacheStats
- -#include <types.h>-
Struct that holds the stats of a KV cache manager.
--Public Members
-
-
-
-SizeType32 maxNumBlocks
Max number of blocks.
-
- +static UnderlyingType constexpr kAuto = {1u << (kNumFlags + 0)}
+-
-
-
-SizeType32 freeNumBlocks
Number of free blocks.
-
- +static UnderlyingType constexpr kTopK = {1u << (kNumFlags + 1)}
+-
-
-
-SizeType32 usedNumBlocks
Number of used blocks.
-
- +static UnderlyingType constexpr kTopP = {1u << (kNumFlags + 2)}
+-
-
-
-SizeType32 tokensPerBlock
Number of tokens per block.
-
- +static UnderlyingType constexpr kBeamSearch = {1u << (kNumFlags + 3)}
+-
-
-
-SizeType32 allocTotalBlocks
Number of total allocated block.
-
- +static UnderlyingType constexpr kMedusa = {1u << (kNumFlags + 4)}
+-
-
-
-SizeType32 allocNewBlocks
Number of newly allocated block.
-
- +static UnderlyingType constexpr kLookahead = {1u << (kNumFlags + 5)}
+-
-
-
-SizeType32 reusedBlocks
Number of reused block.
-
- +static UnderlyingType constexpr kExplicitDraftTokens = {1u << (kNumFlags + 6)}
+ + +-
+
-
+static UnderlyingType constexpr kExternalDraftTokens = {1u << (kNumFlags + 7)}
-
-
-
-SizeType32 missedBlocks
Number of not reused block.
-
- +static UnderlyingType constexpr kEagle = {1u << (kNumFlags + 8)}
+-
-
-
-float cacheHitRate
Measuring the KV Cache reuse rate. cacheHitRate = reusedBlocks / (reusedBlocks + missedBlocks).
-
- +static UnderlyingType constexpr kTopKTopP = {kTopK | kTopP}
+ -
-SizeType32 maxNumBlocks
-
-
-
-struct StaticBatchingStats
-
+struct DisServingRequestStats
- #include <types.h>-
Struct that holds the stats of static batching models for a single iteration.
+Struct that holds the request stats in the case of disaggregated serving.
@@ -2016,129 +2292,65 @@Public Members
-
-
-
-SizeType32 numScheduledRequests
Number of scheduled requests.
-
-
-
-
-SizeType32 numContextRequests
Number of requests in context stage.
-
-
-
-
-SizeType32 numCtxTokens
Total number of context tokens in the iteration.
-
-
-
-
-SizeType32 numGenTokens
Total number of tokens to generate in the iteration.
-
-
-
-
-SizeType32 emptyGenSlots
Total number of unused generation token slots.
+-
+double kvCacheTransferMS
The total time spent on transferring KV cache from context phase to generation phase (ms)
types.h
-
-SizeType32 numScheduledRequests
-
-
-
-struct DisServingRequestStats
- -#include <types.h>-
Struct that holds the request stats in the case of disaggregated serving.
---Public Members
--
-
-
-double kvCacheTransferMS
The total time spent on transferring KV cache from context phase to generation phase (ms)
-
-
-double kvCacheTransferMS
-
-
-
-struct RequestStats
-
+struct KvCacheStats
- #include <types.h>-
Struct that holds the stats of a single request.
+Struct that holds the stats of a KV cache manager.
+Public Members
- --
-
-
-RequestStage stage
The current stage the request is in.
-
-
-
-
-SizeType32 contextPrefillPosition
If using chunked context, the current context prefill position.
-
-
-
-
-SizeType32 numGeneratedTokens
The number of generated tokens so far.
-
-
-
-
-float avgNumDecodedTokensPerIter
The average number of decoded tokens per iteration. It is >= 1 for speculative decoding.
-
-
-
-
-bool scheduled
Whether the request is scheduled for the current iteration.
-
-
-
-
-bool paused
Whether the request is being paused at the current iteration due to lack of resources (KV cache blocks exhaustion for example)
+-
+SizeType32 maxNumBlocks
Max number of blocks.
-
-
-
-std::optional<DisServingRequestStats> disServingStats
Stats specific to disaggregated serving.
+-
+SizeType32 freeNumBlocks
Number of free blocks.
-
-
-
-SizeType32 allocTotalBlocksPerRequest
Number of total allocated blocks per request.
+-
+SizeType32 usedNumBlocks
Number of used blocks.
-
-
-
-SizeType32 allocNewBlocksPerRequest
Number of newly allocated blocks per request.
+-
+SizeType32 tokensPerBlock
Number of tokens per block.
-
-
-
-SizeType32 reusedBlocksPerRequest
Number of reused blocks per request.
+-
+SizeType32 allocTotalBlocks
Number of total allocated block.
-
-
-
-SizeType32 missedBlocksPerRequest
Number of missed blocks per request.
+-
+SizeType32 allocNewBlocks
Number of newly allocated block.
-
-
-
-SizeType32 kvCacheHitRatePerRequest
KV Cache Hit Rate per request, defined as reusedBlocks / (reusedBlocks + missedBlocks)
-
-
-RequestStage stage
-
+SizeType32 reusedBlocks
Number of reused block.
-
-
-
-struct RequestStatsPerIteration
- -#include <types.h>-
Struct that holds the stats of all requests in an iteration.
--@@ -2257,1611 +2469,1630 @@Public Members
-
-
-
-IterationType iter
The iteration id for these stats.
+-
+SizeType32 missedBlocks
Number of not reused block.
-
-
-
-std::vector<RequestStats> requestStats
The stats of all active requests for this iteration.
+-
+float cacheHitRate
Measuring the KV Cache reuse rate. cacheHitRate = reusedBlocks / (reusedBlocks + missedBlocks).
types.h
-
-IterationType iter
-
+inline auto constexpr useTemperature(bool useTemp)
-
+
-
-using TensorPtr = std::shared_ptr<Tensor>
-
+
-
+struct RequestStats
- +#include <types.h>+
Struct that holds the stats of a single request.
+++Public Members
+ + +-
+
-
+RequestStage stage
The current stage the request is in.
+
-
+
-
+SizeType32 contextPrefillPosition
If using chunked context, the current context prefill position.
+
-
+
-
+SizeType32 numGeneratedTokens
The number of generated tokens so far.
+
-
+
-
+float avgNumDecodedTokensPerIter
The average number of decoded tokens per iteration. It is >= 1 for speculative decoding.
+
-
+
-
+bool scheduled
Whether the request is scheduled for the current iteration.
+
-
+
-
+bool paused
Whether the request is being paused at the current iteration due to lack of resources (KV cache blocks exhaustion for example)
+
-
+
-
+std::optional<DisServingRequestStats> disServingStats
Stats specific to disaggregated serving.
+
-
+
-
+SizeType32 allocTotalBlocksPerRequest
Number of total allocated blocks per request.
+
-
+
-
+SizeType32 allocNewBlocksPerRequest
Number of newly allocated blocks per request.
+
-
+
-
+SizeType32 reusedBlocksPerRequest
Number of reused blocks per request.
-
-
-
-TimePoint kvCacheTransferEnd
End time of the KV cache transfer for disaggregated serving.
+-
+SizeType32 missedBlocksPerRequest
Number of missed blocks per request.
-
+
-
+SizeType32 kvCacheHitRatePerRequest
KV Cache Hit Rate per request, defined as reusedBlocks / (reusedBlocks + missedBlocks)
-
+RequestStage stage
-
-namespace executor
-
+enumerator kFIRST_COME_FIRST_SERVED
-
-static constexpr auto value = DataType::kINT8
-
-
-
-struct DebugTensorsPerIteration
-
+struct RequestStatsPerIteration
- #include <types.h>-
Struct that holds the debug tensors in an iteration.
+Struct that holds the stats of all requests in an iteration.
Public Members
-
-
-
-IterationType iter
The iteration id for these tensors.
+-
+IterationType iter
The iteration id for these stats.
-
-
-
-std::map<std::string, Tensor> debugTensors
The debug tensors for this iteration.
+-
+std::vector<RequestStats> requestStats
The stats of all active requests for this iteration.
-
-IterationType iter
-
-
-
-class DecodingMode
-
+struct StaticBatchingStats
- #include <types.h>-
mode of the decoder
--+Public Types
--
-
-
-using UnderlyingType = uint32_t
Struct that holds the stats of static batching models for a single iteration.
-+Public Functions
--
-
-
-inline constexpr auto useTemperature(bool useTemp)
-
-
-
-inline constexpr auto useOccurrencePenalties(bool usePenalty)
-
-
-
-inline constexpr auto usePresencePenalty(bool usePenalty)
Public Members
+-
+
-
+SizeType32 numScheduledRequests
Number of scheduled requests.
+
-
-
-
-inline constexpr auto useRepetitionPenalty(bool usePenalty)
-
+
-
+SizeType32 numContextRequests
Number of requests in context stage.
+
-
-
-
-inline constexpr auto useFrequencyPenalty(bool usePenalty)
-
+
-
+SizeType32 numCtxTokens
Total number of context tokens in the iteration.
+
-
-
-
-inline constexpr auto useMinLength(bool useMinLen)
-
+
-
+SizeType32 numGenTokens
Total number of tokens to generate in the iteration.
+
-
-
-
-inline constexpr auto useBanTokens(bool banTokens)
-
+
-
+SizeType32 emptyGenSlots
Total number of unused generation token slots.
+
-
-
-
-inline constexpr auto useBanWords(bool banWords)
-
-using UnderlyingType = uint32_t
-
-inline constexpr auto useNoRepeatNgramSize(bool noRepeatNgramSize)
-
+template<typename T, bool = false>
struct TypeTraits
+ - +#include <types.h>+
For converting a C++ data type to a
+TrtLmmDataType. -
-inline constexpr auto useStopWords(bool stopWords)
-
+template<>
struct TypeTraits<bool>
+ - +
-
-inline constexpr auto useExplicitEosStop(bool explicitEosStop)
-
+template<>
struct TypeTraits<float>
+ - +
-
-inline constexpr bool isTopK() const
-
+template<>
struct TypeTraits<half>
+ - +
-
-inline constexpr bool isTopKorTopP() const
- +template<> int32_t > +
- +
-
-inline constexpr bool isBeamSearch() const
- +template<> int64_t > +
- +
-
-inline constexpr bool isLookahead() const
- +template<> int8_t > +
- +
-
-inline constexpr bool isExternalDraftTokens() const
- +template<> uint8_t > +
- +
-
-inline constexpr bool isUseTemperature() const
-
-inline constexpr bool isUseFrequencyPenalty() const
-
-inline constexpr bool isUseRepetitionPenalty() const
-
+namespace runtime
-
-inline constexpr bool isUseMinLength() const
-
-inline constexpr bool isUseOccurrencePenalty() const
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
-
-
-
-
+
-
-
-
-
+ executor.h
+-
+
-
+namespace tensorrt_llm
-
+
-
+namespace batch_manager
-
+
-
+namespace kv_cache_manager
-
-
-
-inline constexpr bool isUsePenalty() const
-
+namespace kv_cache_manager
-
-
-
-inline constexpr bool isUseBanWords() const
-
+namespace executor
- ++
Typedefs
+-
+
-
+using RetentionPriority = SizeType32
-
-
-
-inline constexpr bool isUseNoRepeatNgramSize() const
-
+using KVCacheEventData = std::variant<KVCacheCreatedData, KVCacheStoredData, KVCacheRemovedData, KVCacheUpdatedData>
-
+
++Functions
-
-
-
-inline constexpr bool isUseBanTokens() const
- +char const *version() noexcept
+Version of TRT-LLM.
+-
+
-
+struct AdditionalOutput
- + +
-
-
-
-inline bool operator==(DecodingMode const &other) const
-
+class ContextPhaseParams
- ++
Public Types
+-
+
-
+using RequestIdType = std::uint64_t
+-Public Functions
-
-
-
-inline explicit constexpr DecodingMode(UnderlyingType state)
-
+explicit ContextPhaseParams(VecTokens firstGenTokens, RequestIdType reqId)
-
-
-
-inline constexpr UnderlyingType getState() const
-
+ContextPhaseParams(VecTokens firstGenTokens, RequestIdType reqId, void *state)
--Public Static Functions
--
-
-
-static inline constexpr auto Auto()
No mode specified. Config will be determined from the beam width of the first request at runtime TopKTopP if beamWidth == 1, BeamSearch otherwise.
-
-
-
-
-static inline constexpr auto TopK()
-
+ContextPhaseParams(ContextPhaseParams const&)
-
-
-
-static inline constexpr auto TopP()
-
+ContextPhaseParams(ContextPhaseParams&&) noexcept
-
-
-
-static inline constexpr auto TopKTopP()
-
+ContextPhaseParams &operator=(ContextPhaseParams const&)
-
-
-
-static inline constexpr auto BeamSearch()
-
+ContextPhaseParams &operator=(ContextPhaseParams&&) noexcept
-
-
-
-static inline constexpr auto Lookahead()
-
+bool operator==(ContextPhaseParams const&) const noexcept
-
-
-
-static inline constexpr auto ExplicitDraftTokens()
-
+VecTokens const &getFirstGenTokens() const & noexcept
-
-
-
-static inline constexpr auto ExternalDraftTokens()
-
+VecTokens popFirstGenTokens() && noexcept
-
-
-
-static inline constexpr auto Eagle()
-
+RequestIdType getReqId() const noexcept
-Private Functions
-
-
-
-inline constexpr bool anyBitSet(UnderlyingType bits) const
-
+void const *getState() const noexcept
-
-
-
-inline constexpr bool allBitSet(UnderlyingType bits) const
-
+void *getState() noexcept
-
-
-
-inline constexpr UnderlyingType setBitTo(UnderlyingType state, bool x)
-
+void *releaseState() noexcept
-Private Members
--
-
-
-UnderlyingType mState = {}
Private Types
+-+Private Static Attributes
--
-
-
-static constexpr UnderlyingType kUseRepetitionPenalties = {1u << 0}
-
-
-
-static constexpr UnderlyingType kUseFrequencyPenalties = {1u << 1}
-
-
-
-static constexpr UnderlyingType kUsePresencePenalties = {1u << 2}
-
-
-
-static constexpr UnderlyingType kUseTemperature = {1u << 3}
Private Members
-
-
-
-static constexpr UnderlyingType kUseMinLength = {1u << 4}
- +RequestIdType mReqId = {0}
+This request corresponds to the request ID in the context phase.
+-
-
-
-static constexpr UnderlyingType kUseBanWords = {1u << 5}
- +VecTokens mFirstGenTokens
+The first tokens generated by context executor.
+-
-
-
-static constexpr UnderlyingType kUseStopWords = {1u << 6}
- +StatePtr mState = {nullptr, deleter}
+Context phase state of this request.
+-
-
-
-static constexpr UnderlyingType kUseMaxLengthStop = {1u << 7}
++Private Static Functions
+-
+
-
+static void deleter(void const *data)
-
-
-
-static constexpr UnderlyingType kUseExplicitEosStop = {1u << 8}
++Friends
+-
+
- +friend class Serialization +
-
-
-
-static constexpr UnderlyingType kUseNoRepeatNgramSize = {1u << 9}
-
+using RequestIdType = std::uint64_t
-
-static constexpr UnderlyingType kStandardStopCriteria = {kUseStopWords | kUseMaxLengthStop}
-
+class DebugConfig
- +#include <executor.h>+
Configuration class for debugging output.
+++Public Functions
+-
+
-
+explicit DebugConfig(bool debugInputTensors = false, bool debugOutputTensors = false, StringVec debugTensorNames = {}, SizeType32 debugTensorsMaxIterations = 0)
-
-
-
-static constexpr UnderlyingType kUseOccurrencePenalties{kUseRepetitionPenalties | kUseFrequencyPenalties | kUsePresencePenalties}
-
+bool operator==(DebugConfig const &other) const
-
-static constexpr UnderlyingType kUsePenalties = {kUseOccurrencePenalties | kUseTemperature | kUseMinLength}
-
+bool getDebugInputTensors() const
-
-static constexpr UnderlyingType kUseBanTokens = {kUseNoRepeatNgramSize | kUseBanWords}
-
+bool getDebugOutputTensors() const
-
-static constexpr SizeType32 kNumFlags = {10}
-
-static constexpr UnderlyingType kAuto = {1u << (kNumFlags + 0)}
-
+SizeType32 getDebugTensorsMaxIterations() const
-
-static constexpr UnderlyingType kTopK = {1u << (kNumFlags + 1)}
-
+void setDebugInputTensors(bool debugInputTensors)
-
-static constexpr UnderlyingType kTopP = {1u << (kNumFlags + 2)}
-
+void setDebugOutputTensors(bool debugOutputTensors)
-
-static constexpr UnderlyingType kBeamSearch = {1u << (kNumFlags + 3)}
-
-static constexpr UnderlyingType kMedusa = {1u << (kNumFlags + 4)}
-
+void setDebugTensorsMaxIterations(SizeType32 debugTensorsMaxIterations)
-
-static constexpr UnderlyingType kLookahead = {1u << (kNumFlags + 5)}
-
+
-
-
-
+
-
-
-
+
-
-
-
-
-
+
-
-
-
+
-
-
-
+
-
-
-
-
-
+
-
-
++Private Types
+-
+
-
+using StringVec = std::vector<std::string>
+-Private Members
-
-
-
-static constexpr UnderlyingType kExplicitDraftTokens = {1u << (kNumFlags + 6)}
- +bool mDebugInputTensors
+If true, debug all input tensors.
+-
-
-
-static constexpr UnderlyingType kExternalDraftTokens = {1u << (kNumFlags + 7)}
- +bool mDebugOutputTensors
+If true, debug all output tensors.
+-
-
-
-static constexpr UnderlyingType kEagle = {1u << (kNumFlags + 8)}
- +StringVec mDebugTensorNames
+If not empty, only debug tensors in this list.
+-
-
-
-static constexpr UnderlyingType kTopKTopP = {kTopK | kTopP}
- +SizeType32 mDebugTensorsMaxIterations
+If > 0, provide debug tensors for at most debugTensorsMaxIterations past iterations, else dump them to files.
+ -
+explicit DebugConfig(bool debugInputTensors = false, bool debugOutputTensors = false, StringVec debugTensorNames = {}, SizeType32 debugTensorsMaxIterations = 0)
- +friend class Serialization +
-
+
-
-
-
+
+Friends
+-
+
-
+using RetentionPriority = SizeType32
-
-namespace runtime
-
+class DecodingConfig
- +#include <executor.h>+
Configuration class for the decoding.
++Public Functions
+-
+
-
+explicit DecodingConfig(std::optional<DecodingMode> decodingMode = std::nullopt, std::optional<LookaheadDecodingConfig> lookaheadDecodingConfig = std::nullopt, std::optional<MedusaChoices> medusaChoices = std::nullopt, std::optional<EagleConfig> eagleConfig = std::nullopt)
-
+
-
+bool operator==(DecodingConfig const &other) const
-
+
-
+void setDecodingMode(DecodingMode const&)
Sets decoding mode. Some modes require the use of their own setters.
- disaggServerUtil.h
--
-
-
-namespace tensorrt_llm
-
-
-
-namespace executor
-
-
-
-namespace disagg_executor
-
-
-
-struct ResponseWithId
- -
Public Functions
-
-
-
-inline ResponseWithId(tensorrt_llm::executor::Response &&response, IdType gid)
-
+std::optional<DecodingMode> getDecodingMode() const
-
-
-
-inline ResponseWithId(tensorrt_llm::executor::Response const &response, IdType gid)
- +void setLookaheadDecoding(LookaheadDecodingConfig const &lookaheadDecodingConfig)
+Sets lookahead decoding mode and config.
+-
-
-
-inline ResponseWithId(ResponseWithId &&other) noexcept
-
+void enableSeamlessLookaheadDecoding()
-
-
-
-ResponseWithId(ResponseWithId const &other) = default
-
+std::optional<LookaheadDecodingConfig> getLookaheadDecodingConfig() const
-
-
-
-inline ResponseWithId &operator=(ResponseWithId &&other) noexcept
-
+SizeType32 getLookaheadDecodingMaxNumRequest() const
-
-
-
-inline ResponseWithId &operator=(ResponseWithId const &other)
-
+void setMedusaChoices(MedusaChoices const&)
Sets medusa mode and config.
+
-
+
-
+std::optional<MedusaChoices> getMedusaChoices() const
-
-
-
-~ResponseWithId() = default
-
+void setEagleConfig(EagleConfig const&)
Sets eagle mode and config.
+
-
+
-
+std::optional<EagleConfig> getEagleConfig() const
--Public Members
+Private Members
-
-
-
-tensorrt_llm::executor::Response response
-
+std::optional<DecodingMode> mDecodingMode
-
-
-
-IdType gid
-
+std::optional<LookaheadDecodingConfig> mLookaheadDecodingConfig
-
+
-
+std::optional<MedusaChoices> mMedusaChoices
-
+
-
+std::optional<EagleConfig> mEagleConfig
-
-inline ResponseWithId(tensorrt_llm::executor::Response &&response, IdType gid)
++Private Static Attributes
+-
+
-
+static constexpr SizeType32 mLookaheadDecodingMaxNumRequest = 8
-
-
-
-class DisaggExecutorOrchestrator
- -+
Public Functions
--
-
-
-DisaggExecutorOrchestrator(std::vector<std::filesystem::path> const &ctxEnginePaths, std::vector<std::filesystem::path> const &genEnginePaths, std::vector<executor::ExecutorConfig> const &ctxExecutorConfigs, std::vector<executor::ExecutorConfig> const &genExecutorConfigs, bool hasContextAwaitThreads, bool hasGenAwaitThreads)
Constructs a DisaggExecutorOrchestrator object.
--
-
- Parameters: -
-
-
ctxEnginePaths – A vector of file paths to context engine files.
-genEnginePaths – A vector of file paths to generation engine files.
-ctxExecutorConfigs – A vector of ExecutorConfig for context executors.
-genExecutorConfigs – A vector of ExecutorConfig for generation executors.
-hasContextAwaitThreads – Whether or not there are threads that receive response for each generation executor.
-hasGenAwaitThreads – Whether or not there are threads that receive response for each generation executor.
-
+
+Friends
+-
+
- +friend class Serialization
-
-
-
-std::vector<IdType> enqueueContext(std::vector<texec::Request> const &requests, std::optional<int> selectContextId = std::nullopt, bool batch = false)
Enqueue context-only requests to context executors.
--
-
- Parameters: -
-
-
requests – A vector of context-only requests.
-selectContextId – The index of the context executor to use. If
std::nullopt, the executor that has the smallest number of inflight requests will be used.
-batch – If true,enqueue requests in same context executor.If false, will try to use a different executor for each request.
-
-- Returns: -
A vector of global request ids, corresponding to the order of the requests in
-requests, the id returned may be different from the request id in each executor.
-
-
-DisaggExecutorOrchestrator(std::vector<std::filesystem::path> const &ctxEnginePaths, std::vector<std::filesystem::path> const &genEnginePaths, std::vector<executor::ExecutorConfig> const &ctxExecutorConfigs, std::vector<executor::ExecutorConfig> const &genExecutorConfigs, bool hasContextAwaitThreads, bool hasGenAwaitThreads)
-
+
-
+class DynamicBatchConfig
- +#include <executor.h>+
Configuration class for dynamic tuning of batch size and max num tokens. During runtime the statistics of input and output lengths are recoreded. Based on these statistics, the batch size and max num tokens are tuned dynamically to better serve the requests.
+++Public Functions
-
-
-
-void enqueueGeneration(std::vector<texec::Request> const &requests, std::vector<IdType> const &globalRequestIds, std::optional<int> selectGenIdx = std::nullopt, bool batch = false)
Enqueue generation-only requests to generation executors.
--
-
- Parameters: -
-
-
requests – A vector of generation-only requests.
-globalRequestIds – A vector of global request ids, corresponding to the order of the requests,and must be the ids returned by the enqueueContext function.
-selectGenIdx – The index of the generation executor to use. If
std::nullopt, the executor that has the smallest number of inflight requests will be used.
-batch – If true,enqueue requests in same generation executor.If false, will try to use a different executor for each request.
-
-
- +explicit DynamicBatchConfig(bool enableBatchSizeTuning = false, bool enableMaxNumTokensTuning = false, SizeType32 dynamicBatchMovingAverageWindow = kDefaultDynamicBatchMovingAverageWindow, std::vector<std::pair<SizeType32, SizeType32>> batchSizeTable = kDefaultBatchSizeTable)
+-
-
-
-std::vector<ResponseWithId> awaitContextResponses(std::optional<std::chrono::milliseconds> const &timeout, std::optional<int> contextIdx = std::nullopt)
Await for context responses.
--
-
- Parameters: -
-
-
timeout – The maximum time to wait for new responses
-contextIdx – The index of the context executor to use. If
std::nullopt, return ready responses in all context executors,ifhasContextAwaitThreadsis true, then this parameter must be std::nullopt.
-
-- Returns: -
A vector of responses with corresponding global request ids
-
-
- +SizeType32 getDynamicBatchMovingAverageWindow() const
+-
-
-
-std::vector<ResponseWithId> awaitGenerationResponses(std::optional<std::chrono::milliseconds> const &timeout, std::optional<int> genIdx = std::nullopt)
Await for generation responses.
--
-
- Parameters: -
-
-
timeout – The maximum time to wait for new responses.
-genIdx – The index of the generation executor to use. If
std::nullopt, return ready responses in all generation executors,ifhasGenAwaitThreadsis true, then this parameter must be std::nullopt.
-
-- Returns: -
A vector of responses with corresponding global request ids.
-
-
- +bool getEnableBatchSizeTuning() const
+-
-
-
-bool canEnqueue() const
Indicates if the current process is allowed to enqueueRequests.
-
- +bool getEnableMaxNumTokensTuning() const
+-
-
-
-std::vector<std::unique_ptr<texec::Executor>> const &getContextExecutors() const
Get context executors.
-
- +std::vector<std::pair<SizeType32, SizeType32>> getBatchSizeTable() const
+ --
-
-
-std::vector<std::unique_ptr<texec::Executor>> const &getGenExecutors() const
Get generation executors.
+
+Public Static Attributes
+-
+
-
+static SizeType32 const kDefaultDynamicBatchMovingAverageWindow = 128
The default window size for moving average of input and output length which is used to calculate dynamic batch size and max num tokens.
-
-
-
-~DisaggExecutorOrchestrator()
-
+
-
+static std::vector<std::pair<SizeType32, SizeType32>> const kDefaultBatchSizeTable
The default value of batch size table.
+
+Private Members
-
-
-
-std::unique_ptr<Impl> mImpl
-
-void enqueueGeneration(std::vector<texec::Request> const &requests, std::vector<IdType> const &globalRequestIds, std::optional<int> selectGenIdx = std::nullopt, bool batch = false)
-
+bool mEnableBatchSizeTuning
Controls if the batch size should be tuned dynamically.
-
+
-
+bool mEnableMaxNumTokensTuning
Controls if the max num tokens should be tuned dynamically.
-
+
-
+SizeType32 mDynamicBatchMovingAverageWindow
The window size for moving average of input and output length which is used to calculate dynamic batch size and max num tokens.
-
+
-
+std::vector<std::pair<SizeType32, SizeType32>> mBatchSizeTable
A vector of (batchSizeLimit, batchSize). When max capacity batch size is less than.
- tensor.h
--
-
-
-namespace tensorrt_llm
-
+
++Friends
+-
-namespace executor
-
-
-
-class Shape : public tensorrt_llm::common::ArrayView<detail::DimType64 const>
-
-class Shape : public tensorrt_llm::common::ArrayView<detail::DimType64 const>
-
-struct ResponseWithId
-
+
-
+struct EagleConfig
- --
Public Types
--
-
-
-using Base = tensorrt_llm::common::ArrayView<detail::DimType64 const>
-
+explicit EagleConfig(std::optional<EagleChoices> eagleChoices = std::nullopt, bool greedySampling = true, std::optional<float> posteriorThreshold = std::nullopt)
-
-using DimType64 = typename std::remove_cv_t<Base::value_type>
-
+bool operator==(EagleConfig const &other) const
Public Functions
+-
+
-
-
-
+
-+Public Functions
-
-
-
-inline Shape()
-
+std::optional<EagleChoices> getEagleChoices() const
++Private Functions
+-
+
-
+std::optional<float> const &checkPosteriorValue(std::optional<float> const &value)
+@@ -3938,90 +4169,115 @@Private Members
+-
+
-
+std::optional<EagleChoices> mEagleChoices
choices forming tree for EAGLE-1.
-
-
-
-class Tensor
- -+
Public Types
--
-
-
-using CudaStreamPtr = std::shared_ptr<runtime::CudaStream>
-
+
-
+bool mGreedySampling
Flag to use greedy or typical acceptance.
+
-
+
-
+std::optional<float> mPosteriorThreshold
Minimum token probability of the typical acceptance. Corresponds to epsilon in https://arxiv.org/pdf/2401.10774. Default is 0.09f.
+
++Friends
+-
+
- +friend class Serialization +
-
-using CudaStreamPtr = std::shared_ptr<runtime::CudaStream>
-
+
-
+class Executor
- +#include <executor.h>+
The executor is responsible for receiving new requests and sending responses, and running the inference.
-Public Functions
-
-
-
-Tensor copyToCpu(Tensor::CudaStreamPtr stream = nullptr) const
- +Executor(std::filesystem::path const &modelPath, ModelType modelType, ExecutorConfig const &executorConfig)
+-
+
- Parameters: +
-
+
modelPath – Path to the folder that defines the model to run
+modelType – The type of model
+executorConfig – The configuration for the executor
+
+
-
-
-
-Tensor copyToPinned(Tensor::CudaStreamPtr stream = nullptr) const
-
+Executor(std::filesystem::path const &encoderModelPath, std::filesystem::path const &decoderModelPath, ModelType modelType, ExecutorConfig const &executorConfig)
-
-
-
-Tensor copyToPooledPinned(Tensor::CudaStreamPtr stream = nullptr) const
-
+Executor(BufferView const &engineBuffer, std::string const &jsonConfigStr, ModelType modelType, ExecutorConfig const &executorConfig, std::optional<std::map<std::string, Tensor>> const &managedWeights = std::nullopt)
-
-
-
-Tensor copyToManaged(Tensor::CudaStreamPtr stream = nullptr) const
-
+Executor(BufferView const &encoderEngineBuffer, std::string const &encoderJsonConfigStr, BufferView const &decoderEngineBuffer, std::string const &decoderJsonConfigStr, ModelType modelType, ExecutorConfig const &executorConfig)
-
-
-
-Tensor copyToGpu(Tensor::CudaStreamPtr stream) const
-
-
-
-Tensor() noexcept = default
-
-
-
-Tensor(Tensor &&other) noexcept = default
-
+Executor &operator=(Executor const &executor) = delete
-
-
-
-Tensor &operator=(Tensor &&other) noexcept = default
-
+Executor &operator=(Executor&&) = default
-
-
-
-void *getData()
Returns a pointer to underlying array.
-
-
-
-
-void const *getData() const
Returns a pointer to underlying array.
-
-
-
-
-MemoryType getMemoryType() const
Returns the memory type of the buffer.
-
-
-
-
-std::size_t getSize() const
Returns the number of elements in the tensor.
-
-
-
-
-std::size_t getSizeInBytes() const
Returns the size of the tensor in bytes.
-
-
-
-
-void setZero(CudaStreamPtr stream = nullptr)
Set the entire memory to zero.
+-
+IdType enqueueRequest(Request const &request)
Enqueue a new request.
- Parameters: -
stream – Must be a valid CUDA stream if the memory type is GPU.
+request – The LLM request which contains input tokens and request parameters
-
-
-
-void setFrom(Tensor const &other, CudaStreamPtr stream = nullptr)
Copy the data and shape from another tensor.
--
-
- Parameters: -
-
-
other – A tensor to copy from.
-stream – Must be a valid CUDA stream if the memory type is GPU.
-
- Returns: +
A unique id that identifies the request
-
-
-
-inline explicit operator bool() const
- +std::vector<IdType> enqueueRequests(std::vector<Request> const &requests)
+Enqueue a batch of request.
+--Public Static Functions
-
-
-
-static Tensor cpu(DataType dataType, Shape shape = {})
Allocate a cpu tensor with the given shape and data type.
+-
+std::vector<Response> awaitResponses(std::optional<std::chrono::milliseconds> const &timeout = std::nullopt)
Await for ready responses.
+++This overload awaits for any ready responses. In particular, if several requests + have been enqueued, this method will provide any ready responses without order guarantees. +
- Parameters: -
-
-
shape – The shape of the tensor.
-dataType – The data type of the tensor.
-
timeout – The maximum time to wait for new responses
+- Returns: +
A vector of responses
-
-
-
-static Tensor pinned(DataType dataType, Shape shape = {})
Allocate a cpu tensor in pinned memory with the given shape and data type.
+-
+std::vector<Response> awaitResponses(IdType const &requestId, std::optional<std::chrono::milliseconds> const &timeout = std::nullopt)
Await for ready responses.
- Parameters:
-
-
shape – The shape of the tensor.
-dataType – The data type of the tensor.
+id – A request id
+timeout – The maximum time to wait for new responses
+- Returns: +
A vector of responses
+
-
-
-
-static Tensor pooledPinned(DataType dataType, Shape shape = {})
Allocate a cpu tensor in pooled pinned memory with the given shape and data type.
+-
+std::vector<std::vector<Response>> awaitResponses(std::vector<IdType> const &requestIds, std::optional<std::chrono::milliseconds> const &timeout = std::nullopt)
Await for multiple ready responses.
+++A multiple ID request behaves as if awaitResponses(IdType, timeout) + were invoked on all IDs. The returned vector contains + a vector of responses per ID in the same order specified by the requestIds. + The same behaviour as awaitResponses(IdType, timeout) applies: + * Responses may be empty. + * If all responses have already been given for one of the requestIds, + then this method will hang unless a timeout is specified. +
- Parameters:
-
-
shape – The shape of the tensor.
-dataType – The data type of the tensor.
+requestIds – Ids requested
+timeout – The maximum time to wait for new responses
+- Returns: +
A vector of vector of responses
+
-
-
-
-static Tensor managed(DataType dataType, Shape shape = {})
Allocate a tensor in managed memory (UVM) with the given shape and data type.
+-
+SizeType32 getNumResponsesReady(std::optional<IdType> const &requestId = std::nullopt) const
Get the number of ready responses.
- Parameters: -
-
-
shape – The shape of the tensor.
-dataType – The data type of the tensor.
-
requestId – An optional request id
+- Returns: +
The number of ready responses
-
-
-
-static Tensor gpu(DataType dataType, CudaStreamPtr stream, Shape shape = {})
Allocate a gpu tensor with the given shape and data type on a particular cuda stream.
+-
+void cancelRequest(IdType requestId)
Cancel the request with provided request id.
- Parameters: -
-
-
shape – The shape of the tensor.
-stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.
-dataType – The data type of the tensor.
-
id – The request id for which to cancel the response
-
-
-
-template<typename T>
static inline Tensor gpu(CudaStreamPtr stream, Shape shape = {})
-
- +void shutdown()
+Signals the server to shutdown.
+This call is blocking. Only returns when all requests have terminated or timeout has been reached
+-
-
-
-static Tensor of(DataType dataType, void *data, Shape shape)
Wrap a data pointer into a tensor without taking ownership.
+-
+std::deque<IterationStats> getLatestIterationStats()
Returns the per-iterations statistics computed since last call to getLatestIterationStats. Contains at most iterStatsMaxIterations iterations.
-
-
- Parameters: -
-
-
shape – The shape of the tensor.
-dataType – The data type of the tensor.
-stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.
-
- Returns: +
Iteration stats
-
-
-
-template<typename T>
static inline Tensor of(T *data, Shape shape)
- Wrap a data pointer into a tensor without taking ownership.
+-
+std::deque<RequestStatsPerIteration> getLatestRequestStats()
Returns the request stats of each iteration computed since last call to getLatestRequestStats. Contains at most requestStatsMaxIterations iterations.
-
-
- Parameters: -
-
-
shape – The shape of the tensor.
-dataType – The data type of the tensor.
-stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.
-
- Returns: +
Request stats grouped by iterations
-
-
-
-template<typename T>
static inline Tensor of(T &data)
- Wrap any container into a tensor without taking ownership.
+-
+std::deque<DebugTensorsPerIteration> getLatestDebugTensors()
Returns the debug tensors of each iteration computed since last call to getLatestDebugTensors. Contains at most debugTensorsMaxIterations iterations.
-
-
- Parameters: -
-
-
shape – The shape of the tensor.
-dataType – The data type of the tensor.
-stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.
-
- Returns: +
Request debug tensors grouped by iterations
--Private Types
- +-
+
-
+bool canEnqueueRequests() const
Indicates if the current process is allowed to enqueueRequests.
+
-- +Private Functions
-
-
-
- +bool isParticipant() const
+Indicates if the current process participates in this executor instance.
+-
-
+
-
+std::optional<std::shared_ptr<KVCacheEventManager>> getKVCacheEventManager() const
-
+
-
+class ExecutorConfig
- +#include <executor.h>+
Configuration class for the model executor.
--Friends
--
-
- -friend class Serialization -
Public Functions
-
-
-
-friend std::shared_ptr<runtime::ITensor> const &toITensor(Tensor const &tensor)
-
+explicit ExecutorConfig(SizeType32 maxBeamWidth = 1, SchedulerConfig schedulerConfig = SchedulerConfig(), KvCacheConfig kvCacheConfig = KvCacheConfig(), bool enableChunkedContext = true, bool normalizeLogProbs = true, SizeType32 iterStatsMaxIterations = kDefaultIterStatsMaxIterations, SizeType32 requestStatsMaxIterations = kDefaultRequestStatsMaxIterations, BatchingType batchingType = BatchingType::kINFLIGHT, std::optional<SizeType32> maxBatchSize = std::nullopt, std::optional<SizeType32> maxNumTokens = std::nullopt, std::optional<ParallelConfig> parallelConfig = std::nullopt, std::optional<PeftCacheConfig> const &peftCacheConfig = std::nullopt, std::optional<LogitsPostProcessorConfig> logitsPostProcessorConfig = std::nullopt, std::optional<DecodingConfig> decodingConfig = std::nullopt, float gpuWeightsPercent = 1, std::optional<SizeType32> maxQueueSize = std::nullopt, ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig = ExtendedRuntimePerfKnobConfig(), std::optional<DebugConfig> debugConfig = std::nullopt, SizeType32 recvPollPeriodMs = 0, uint64_t maxSeqIdleMicroseconds = kDefaultMaxSeqIdleMicroseconds, std::optional<SpeculativeDecodingConfig> specDecConfig = std::nullopt, std::optional<GuidedDecodingConfig> guidedDecodingConfig = std::nullopt, std::optional<std::vector<std::string>> additionalOutputNames = std::nullopt)
-
-
-
-
-
-
-namespace detail
- -- -
Typedefs
--
-
-
-using DimType64 = int64_t
-
+SizeType32 getMaxBeamWidth() const
-
-using DimType64 = int64_t
-
-Tensor copyToCpu(Tensor::CudaStreamPtr stream = nullptr) const
-
-
-
-namespace runtime
-
+KvCacheConfig getKvCacheConfig() const
- executor.h
--
-
-
-namespace tensorrt_llm
-
-
-
-namespace batch_manager
-
-
-
-namespace kv_cache_manager
-
+SchedulerConfig &getSchedulerConfigRef()
-
+
-
-namespace kv_cache_manager
-
-
-
-namespace executor
- --
Typedefs
--
-
-
-using RetentionPriority = SizeType32
-
+KvCacheConfig &getKvCacheConfigRef()
-
-using KVCacheEventData = std::variant<KVCacheCreatedData, KVCacheStoredData, KVCacheRemovedData, KVCacheUpdatedData>
-
+bool getEnableChunkedContext() const
-
+
-
-
-
+
--Functions
-
-
-
-char const *version() noexcept
Version of TRT-LLM.
-
- +bool getNormalizeLogProbs() const
+ --
-
-
-class SamplingConfig
- -#include <executor.h>-
Sampling configuration.
--Public Functions
-
-
-
-explicit SamplingConfig(SizeType32 beamWidth = 1, std::optional<SizeType32> const &topK = std::nullopt, std::optional<FloatType> const &topP = std::nullopt, std::optional<FloatType> const &topPMin = std::nullopt, std::optional<TokenIdType> const &topPResetIds = std::nullopt, std::optional<FloatType> const &topPDecay = std::nullopt, std::optional<RandomSeedType> const &seed = std::nullopt, std::optional<FloatType> const &temperature = std::nullopt, std::optional<SizeType32> const &minTokens = std::nullopt, std::optional<FloatType> const &beamSearchDiversityRate = std::nullopt, std::optional<FloatType> const &repetitionPenalty = std::nullopt, std::optional<FloatType> const &presencePenalty = std::nullopt, std::optional<FloatType> const &frequencyPenalty = std::nullopt, std::optional<FloatType> const &lengthPenalty = std::nullopt, std::optional<SizeType32> const &earlyStopping = std::nullopt, std::optional<SizeType32> const &noRepeatNgramSize = std::nullopt, std::optional<SizeType32> const &numReturnSequences = std::nullopt)
Constructor for SamplingConfig See description of parameters below.
-
- +SizeType32 getIterStatsMaxIterations() const
+-
-
-
-bool operator==(SamplingConfig const &other) const
-
+SizeType32 getRequestStatsMaxIterations() const
-
-
-
-SizeType32 getBeamWidth() const
-
+BatchingType getBatchingType() const
-
-
-
-SizeType32 getNumReturnBeams() const
-
+std::optional<SizeType32> getMaxBatchSize() const
-
-
-
-std::optional<SizeType32> getTopK() const
-
+std::optional<SizeType32> getMaxNumTokens() const
-
-
-
-std::optional<FloatType> getTopP() const
-
+std::optional<ParallelConfig> getParallelConfig() const
-
-
-
-std::optional<FloatType> getTopPMin() const
-
+std::optional<PeftCacheConfig> getPeftCacheConfig() const
-
-
-
-std::optional<SizeType32> getTopPResetIds() const
-
+std::optional<LogitsPostProcessorConfig> getLogitsPostProcessorConfig() const
-
-
-
-std::optional<FloatType> getTopPDecay() const
-
+std::optional<DecodingConfig> getDecodingConfig() const
-
-
-
-std::optional<RandomSeedType> getSeed() const
-
+float getGpuWeightsPercent() const
-
-
-
-std::optional<RandomSeedType> getRandomSeed() const
-
+std::optional<SizeType32> getMaxQueueSize() const
-
-
-
-std::optional<FloatType> getTemperature() const
-
+ExtendedRuntimePerfKnobConfig getExtendedRuntimePerfKnobConfig() const
-
-
-
-std::optional<SizeType32> getMinTokens() const
-
+std::optional<DebugConfig> getDebugConfig() const
-
-
-
-std::optional<SizeType32> getMinLength() const
-
+SizeType32 getRecvPollPeriodMs() const
-
-
-
-std::optional<FloatType> getBeamSearchDiversityRate() const
-
+uint64_t getMaxSeqIdleMicroseconds() const
-
-
-
-std::optional<FloatType> getRepetitionPenalty() const
-
+std::optional<SpeculativeDecodingConfig> getSpecDecConfig() const
-
-
-
-std::optional<FloatType> getPresencePenalty() const
-
+std::optional<GuidedDecodingConfig> getGuidedDecodingConfig() const
-
-
-
-std::optional<FloatType> getFrequencyPenalty() const
-
+std::optional<std::vector<std::string>> getAdditionalOutputNames() const
-
-
-
-std::optional<FloatType> getLengthPenalty() const
-
+void setMaxBeamWidth(SizeType32 maxBeamWidth)
-
-
-
-std::optional<SizeType32> getEarlyStopping() const
-
+void setMaxBatchSize(SizeType32 maxBatchSize)
-
-
-
-std::optional<SizeType32> getNoRepeatNgramSize() const
-
+void setMaxNumTokens(SizeType32 maxNumTokens)
-
-
-
-std::optional<SizeType32> getNumReturnSequences() const
-
+void setSchedulerConfig(SchedulerConfig const &schedulerConfig)
-
-
-
-void setBeamWidth(SizeType32 beamWidth)
-
+void setKvCacheConfig(KvCacheConfig const &kvCacheConfig)
-
-
-
-void setTopK(std::optional<SizeType32> const &topK)
-
+void setEnableChunkedContext(bool enableChunkedContext)
-
-
-
-void setTopP(std::optional<FloatType> const &topP)
-
+void setNormalizeLogProbs(bool normalizeLogProbs)
-
-
-
-void setTopPMin(std::optional<FloatType> const &topPMin)
-
+void setIterStatsMaxIterations(SizeType32 iterStatsMaxIterations)
-
-
-
-void setTopPResetIds(std::optional<TokenIdType> const &topPResetIds)
-
+void setRequestStatsMaxIterations(SizeType32 requestStatsMaxIterations)
-
-
-
-void setTopPDecay(std::optional<FloatType> const &topPDecay)
-
+void setBatchingType(BatchingType batchingType)
-
-
-
-void setSeed(std::optional<RandomSeedType> const &seed)
-
+void setParallelConfig(ParallelConfig const ¶llelConfig)
-
-
-
-void setRandomSeed(std::optional<RandomSeedType> const &randomSeed)
-
+void setPeftCacheConfig(PeftCacheConfig const &peftCacheConfig)
-
-
-
-void setTemperature(std::optional<FloatType> const &temperature)
-
+void setLogitsPostProcessorConfig(LogitsPostProcessorConfig const &logitsPostProcessorConfig)
-
-
-
-void setMinTokens(std::optional<SizeType32> const &minTokens)
-
+void setDecodingConfig(DecodingConfig const &decodingConfig)
-
-
-
-void setMinLength(std::optional<SizeType32> const &minLength)
-
+void setGpuWeightsPercent(float const &gpuWeightsPercent)
-
-
-
-void setBeamSearchDiversityRate(std::optional<FloatType> const &beamSearchDiversityRate)
-
+void setMaxQueueSize(std::optional<SizeType32> const &maxQueueSize)
-
-
-
-void setRepetitionPenalty(std::optional<FloatType> const &repetitionPenalty)
-
+void setExtendedRuntimePerfKnobConfig(ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig)
-
-
-
-void setPresencePenalty(std::optional<FloatType> const &presencePenalty)
-
+void setDebugConfig(DebugConfig const &debugConfig)
-
-
-
-void setFrequencyPenalty(std::optional<FloatType> const &frequencyPenalty)
-
+void setRecvPollPeriodMs(SizeType32 const &recvPollPeriodMs)
-
-
-
-void setLengthPenalty(std::optional<FloatType> const &lengthPenalty)
-
+void setMaxSeqIdleMicroseconds(uint64_t maxNumTokens)
-
-
-
-void setEarlyStopping(std::optional<SizeType32> const &earlyStopping)
-
+void setSpecDecConfig(SpeculativeDecodingConfig const &specDecConfig)
-
-
-
-void setNoRepeatNgramSize(std::optional<SizeType32> const &noRepeatNgramSize)
-
+void setGuidedDecodingConfig(GuidedDecodingConfig const &guidedDecodingConfig)
-
-
-
-void setNumReturnSequences(std::optional<SizeType32> const &numReturnSequences)
-
+void setAdditionalOutputNames(std::vector<std::string> const &additionalOutputNames)
-Private Functions
--
-
-
-void updateNumReturnBeams()
-
+static constexpr uint64_t kDefaultMaxSeqIdleMicroseconds = 180000000
-
+static constexpr SizeType32 kDefaultIterStatsMaxIterations = 1000
-
+static constexpr SizeType32 kDefaultRequestStatsMaxIterations = 0
Public Static Attributes
+-
+
-
+
-
+
Private Members
-
-
-
-SizeType32 mBeamWidth
The beam width. Default is 1 which disables beam search.
+-
+SizeType32 mMaxBeamWidth
The beam width value of requests that will be sent to the executor.
-
-
-
-std::optional<SizeType32> mTopK
Controls number of logits to sample from. Default is 0 (all logits).
+-
+SchedulerConfig mSchedulerConfig
The scheduler configuration.
-
-
-
-std::optional<FloatType> mTopP
Controls the top-P probability to sample from. Default is 0.f.
+-
+KvCacheConfig mKvCacheConfig
The KV cache configuration.
-
-
-
-std::optional<FloatType> mTopPMin
Controls decay in the top-P algorithm. topPMin is lower-bound. Default is 1.e-6.
+-
+bool mEnableChunkedContext
The KV cache configuration.
-
-
-
-std::optional<TokenIdType> mTopPResetIds
Controls decay in the top-P algorithm. Indicates where to reset the decay. Default is 1.
+-
+bool mNormalizeLogProbs
Controls if log probabilities should be normalized or not.
-
-
-
-std::optional<FloatType> mTopPDecay
Controls decay in the top-P algorithm. The decay value. Default is 1.f.
+-
+SizeType32 mIterStatsMaxIterations
Controls the maximum number of iterations for which to keep statistics.
-
-
-
-std::optional<RandomSeedType> mSeed
Controls the random seed used by the random number generator in sampling.
+-
+SizeType32 mRequestStatsMaxIterations
Controls the maximum number of iterations for which to keep per-request statistics.
-
-
-
-std::optional<FloatType> mTemperature
Controls the modulation of logits when sampling new tokens. It can have values > 0.f. Default is 1.0f.
+-
+BatchingType mBatchingType
The type of batching strategy to use. See BatchingType.
-
-
-
-std::optional<SizeType32> mMinTokens
Lower bound on the number of tokens to generate. Values < 1 have no effect. Default is 1.
+-
+std::optional<SizeType32> mMaxBatchSize
The max batch size of requests.
+
-
+
-
+std::optional<SizeType32> mMaxNumTokens
The max number of tokens per batch.
+
-
+
-
+std::optional<ParallelConfig> mParallelConfig
The parallel execution configuration.
+
-
+
-
+std::optional<PeftCacheConfig> mPeftCacheConfig
-
+
-
+std::optional<LogitsPostProcessorConfig> mLogitsPostProcessorConfig
Logits post processor configuration.
+
-
+
-
+std::optional<DecodingConfig> mDecodingConfig
Decoding configuration.
-
-
-
-std::optional<FloatType> mBeamSearchDiversityRate
Controls the diversity in beam search.
+-
+float mGpuWeightsPercent
GPU weights percent for weight streaming.
-
-
-
-std::optional<FloatType> mRepetitionPenalty
Used to penalize tokens based on how often they appear in the sequence. It can have any value > 0.f. Values < 1.f encourages repetition, values > 1.f discourages it. Default is 1.f.
+-
+std::optional<SizeType32> mMaxQueueSize
The maximum number of requests allowed in queue before rejecting new requests.
-
-
-
-std::optional<FloatType> mPresencePenalty
Used to penalize tokens already present in the sequence (irrespective of the number of appearances). It can have any values. Values < 0.f encourage repetition, values > 0.f discourage it. Default is 0.f.
+-
+ExtendedRuntimePerfKnobConfig mExtendedRuntimePerfKnobConfig
Config for perf knobs that can be set in runtime.
-
-
-
-std::optional<FloatType> mFrequencyPenalty
Used to penalize tokens already present in the sequence (dependent on the number of appearances). It can have any values. Values < 0.f encourage repetition, values > 0.f discourage it. Default is 0.f.
+-
+std::optional<DebugConfig> mDebugConfig
Debugging configuration.
-
-
-
-std::optional<FloatType> mLengthPenalty
Controls how to penalize longer sequences in beam search. Default is 0.f.
+-
+SizeType32 mRecvPollPeriodMs
The time in ms between polls for new communication in orchestrator mode. Use 0 for busy loop.
-
-
-
-std::optional<SizeType32> mEarlyStopping
Controls whether the generation process finishes once beamWidth sentences are generated (ends with end_token)
+-
+uint64_t mMaxSeqIdleMicroseconds
The maximum time in microseconds a scheduled request can remain idle before getting terminated. Default is 3 minutes.
-
-
-
-std::optional<SizeType32> mNoRepeatNgramSize
Controls how many repeat ngram size are acceptable. Default is 1 << 30.
+-
+std::optional<SpeculativeDecodingConfig> mSpeculativeDecodingConfig
The speculative decoding configuration.
-
-
-
-std::optional<SizeType32> mNumReturnSequences
The number of return sequences or beams. In beam search, the value should be less than or equal to mBeamWidth. In sampling, it specifies the total number of independently generated sequences.
+-
+std::optional<GuidedDecodingConfig> mGuidedDecodingConfig
The guided decoding configuration.
-
-
-
-SizeType32 mNumReturnBeams
The number of beams to return. It is equal to beamWidth unless numReturnSequences is set. If beamWidth > 1 and numReturnSequences is set, then numReturnBeams is equal to numReturnSequences.
+-
+std::optional<std::vector<std::string>> mAdditionalOutputNames
The additional output tensor names.
-+Private Static Functions
--
-
-
-static SizeType32 checkBeamWidth(SizeType32 beamWidth)
Friends
+-
+
- +friend class Serialization +
-
-explicit SamplingConfig(SizeType32 beamWidth = 1, std::optional<SizeType32> const &topK = std::nullopt, std::optional<FloatType> const &topP = std::nullopt, std::optional<FloatType> const &topPMin = std::nullopt, std::optional<TokenIdType> const &topPResetIds = std::nullopt, std::optional<FloatType> const &topPDecay = std::nullopt, std::optional<RandomSeedType> const &seed = std::nullopt, std::optional<FloatType> const &temperature = std::nullopt, std::optional<SizeType32> const &minTokens = std::nullopt, std::optional<FloatType> const &beamSearchDiversityRate = std::nullopt, std::optional<FloatType> const &repetitionPenalty = std::nullopt, std::optional<FloatType> const &presencePenalty = std::nullopt, std::optional<FloatType> const &frequencyPenalty = std::nullopt, std::optional<FloatType> const &lengthPenalty = std::nullopt, std::optional<SizeType32> const &earlyStopping = std::nullopt, std::optional<SizeType32> const &noRepeatNgramSize = std::nullopt, std::optional<SizeType32> const &numReturnSequences = std::nullopt)
-
+
-
+class ExtendedRuntimePerfKnobConfig
- +#include <executor.h>+
Configuration class for the runtime perf knobs.
++-Public Functions
-
-
-
-static std::optional<FloatType> const &checkTopPMin(std::optional<FloatType> const &topPMin)
-
+explicit ExtendedRuntimePerfKnobConfig(bool multiBlockMode = true, bool enableContextFMHAFP32Acc = false, bool cudaGraphMode = false, SizeType32 cudaGraphCacheSize = 0)
-
-
-
-static std::optional<TokenIdType> const &checkTopPResetIds(std::optional<TokenIdType> const &topPResetIds)
-
+inline bool operator==(ExtendedRuntimePerfKnobConfig const &other) const
-
-
-
-static std::optional<FloatType> const &checkTopPDecay(std::optional<FloatType> const &topPDecay)
-
+bool getMultiBlockMode() const
-
-
-
-static std::optional<FloatType> const &checkTemperature(std::optional<FloatType> const &temperature)
-
+bool getEnableContextFMHAFP32Acc() const
-
-
-
-static std::optional<FloatType> const &checkRepetitionPenalty(std::optional<FloatType> const &penalty)
-
+bool getCudaGraphMode() const
-
-
-
-static std::optional<SizeType32> const &checkMinTokens(std::optional<SizeType32> const &minTokens)
-
+SizeType32 getCudaGraphCacheSize() const
-
-
-
-static std::optional<SizeType32> const &checkNoRepeatNgramSize(std::optional<SizeType32> const &noRepeatNgramSize)
-
+void setMultiBlockMode(bool multiBlockMode)
-
-
-
-static std::optional<FloatType> const &checkBeamSearchDiversityRate(std::optional<FloatType> const &beamSearchDiversityRate)
-
+void setEnableContextFMHAFP32Acc(bool enableContextFMHAFP32Acc)
-
-
-
-static std::optional<SizeType32> const &checkNumReturnSequences(std::optional<SizeType32> const &numReturnSequences, SizeType32 beamWidth)
-
+void setCudaGraphMode(bool cudaGraphMode)
--Friends
--
-
- -friend class Serialization -
-
-static std::optional<FloatType> const &checkTopPMin(std::optional<FloatType> const &topPMin)
-
-
-
-class OutputConfig
- -#include <executor.h>-
Configuration that controls the outputs of a Result.
--Public Functions
-
-
-
-explicit OutputConfig(bool returnLogProbs = false, bool returnContextLogits = false, bool returnGenerationLogits = false, bool excludeInputFromOutput = false, bool returnEncoderOutput = false, bool returnPerfMetrics = false)
-
+void setCudaGraphCacheSize(SizeType32 cacheSize)
-+Public Members
- - +Private Members
-
-
-
-bool returnContextLogits
Controls if Result should contain the context logits. Default is false.
+-
+bool mMultiBlockMode
Control if multi block mode should be enabled or not.
-
-
-
-bool returnGenerationLogits
Controls if Result should contain the generation logits. Default is false.
+-
+bool mEnableContextFMHAFP32Acc
If enable FMHA runner FP32 accumulation.
-
-
-
-bool excludeInputFromOutput
Controls if output tokens in Result should include the input tokens. Default is false.
+-
+bool mCudaGraphMode
Control if enable cuda graph.
-
-
-
-bool returnEncoderOutput
Controls if Result should contain encoder output hidden states (for encoder-only and encoder-decoder models). Default is false.
+-
+SizeType32 mCudaGraphCacheSize
Number of cuda graphs to be cached in the runtime. The larger the cache, the better the perf, but more GPU memory is consumed.
+Friends
+-
+
- +friend class Serialization +
-
-explicit OutputConfig(bool returnLogProbs = false, bool returnContextLogits = false, bool returnGenerationLogits = false, bool excludeInputFromOutput = false, bool returnEncoderOutput = false, bool returnPerfMetrics = false)
executor.h
- -std::optional<Tensor> getLogits() const
+std::optional<Tensor> getLogits() const
-
-using RetentionPriority = SizeType32
-
-namespace batch_manager
executor.h -
- -class PromptTuningConfig
+- +class GuidedDecodingConfig
#include <executor.h>-Configuration for prompt tuning.
+Guided decoding configurations for executor.
++Public Types
+ + +-Public Functions
-
-
-
-explicit PromptTuningConfig(Tensor embeddingTable, std::optional<VecTokenExtraIds> inputTokenExtraIds = std::nullopt)
-
+explicit GuidedDecodingConfig(GuidedDecodingBackend backend, std::optional<std::vector<std::string>> encodedVocab = std::nullopt, std::optional<std::string> tokenizerStr = std::nullopt, std::optional<std::vector<TokenIdType>> stopTokenIds = std::nullopt)
-
-
-
-Tensor getEmbeddingTable() const
-
+bool operator==(GuidedDecodingConfig const &other) const
-
-
-
-std::optional<VecTokenExtraIds> getInputTokenExtraIds() const
-
+void setBackend(GuidedDecodingBackend const &backend)
--Private Members
--
-
-
-Tensor mEmbeddingTable
The prompt embedding table. Expected shape: [task vocab_size, hidden_size]. Data type must match model weights.
-
-
+
-
+GuidedDecodingBackend getBackend() const
-
-
-
-std::optional<VecTokenExtraIds> mInputTokenExtraIds
The input token extra ids for KV Cache reuse when p-tuning is enabled.
-
-
+
-
+void setEncodedVocab(std::vector<std::string> const &encodedVocab)
--Friends
--
-
- -friend class Serialization -
-
+
-
+std::optional<std::vector<std::string>> getEncodedVocab() const
-
-using Base = tensorrt_llm::common::ArrayView<detail::DimType64 const>
-
+
-
+void setTokenizerStr(std::string const &tokenizerStr)
-
-
-
-class MropeConfig
- -#include <executor.h>-
Configuration for mrope.
--Public Functions
-
-
-
-explicit MropeConfig(Tensor mropeRoratySinCos, SizeType32 mropePositionDeltas)
-
+std::optional<std::string> getTokenizerStr() const
-
+
-
+void setStopTokenIds(std::vector<TokenIdType> const &stopTokenIds)
-
-
-
-Tensor getMRopeRotarySinCos() const
-
+std::optional<std::vector<TokenIdType>> getStopTokenIds() const
-
-
-
-SizeType32 getMRopePositionDeltas() const
-
+void validate() const
@@ -4036,143 +4292,80 @@Private Members
-
-
-
-Tensor mMRopeRotarySinCos
The mrope rotary sin and cos cache. Expected shape: [maxPositionEmbeddings*rotaryEmbeddingDim],Data type must float32.
+-
+GuidedDecodingBackend mBackend
Guided decoding backend. Currently supports XGrammar.
-
-
-
-SizeType32 mMRopePositionDeltas
The mrope position deltas.
+-
+std::optional<std::vector<std::string>> mEncodedVocab
Encoded vocabulary. For a huggingface tokenizer, it can be extracted by:
+++encoded_vocab = tokenizer.get_vocab() +encoded_vocab = [token for token, _ in sorted(encoded_vocab.items(), key=lambda x: x[1])] +
-
+
-
+std::optional<std::string> mTokenizerStr
Tokenizer string. For a huggingface fast tokenizer, it can be extracted by:
+++tokenizer_str = tokenizer.backend_tokenizer.to_str() +
-
+
-
+std::optional<std::vector<TokenIdType>> mStopTokenIds
Stop token ids. If not provided, it can be automatically detected.
executor.h -
- -class LoraConfig
+- +class GuidedDecodingParams
#include <executor.h>-Configuration for LoRA.
+Guided decoding parameters for a request.
--Public Functions
--
-
-
-explicit LoraConfig(IdType taskId, std::optional<Tensor> weights = std::nullopt, std::optional<Tensor> config = std::nullopt)
Public Types
+-Private Members
--
-
-
-IdType mTaskId
The Lora task id.
+-
+
-
+enumerator kREGEX
The generated text is amenable to the user-specified regular expression.
-
-
-
-std::optional<Tensor> mWeights
The Lora weights. See TRT-LLM documentation for expected shapes and types.
+-
+
-
+enumerator kEBNF_GRAMMAR
The generated text is amenable to the user-specified extended Backus-Naur form (EBNF) grammar. EBNF grammar is widely-used to express context-free grammars.
-
+enumerator kEBNF_GRAMMAR
-
+enumerator kREGEX
--Friends
--
-
- -friend class Serialization -
-
-explicit MropeConfig(Tensor mropeRoratySinCos, SizeType32 mropePositionDeltas)
-
-
-
-struct LookaheadDecodingConfig
- -
Public Functions
-
-
-
-LookaheadDecodingConfig(SizeType32 windowSize, SizeType32 ngramSize, SizeType32 verificationSetSize)
-
-
-
-inline explicit LookaheadDecodingConfig()
-
-
-
-bool operator==(LookaheadDecodingConfig const &other) const
-
-
-
-std::tuple<SizeType32 const, SizeType32 const, SizeType32 const> get() const
-
+explicit GuidedDecodingParams(GuideType guideType, std::optional<std::string> guide = std::nullopt)
-
-
-
-SizeType32 getWindowSize() const
-
+bool operator==(GuidedDecodingParams const &other) const
-
-
-
-SizeType32 getNgramSize() const
-
+GuideType getGuideType() const
-
-
-
-SizeType32 getVerificationSetSize() const
-
+std::optional<std::string> getGuide() const
-
-
-
-std::tuple<SizeType32, SizeType32, SizeType32, SizeType32> calculateSpeculativeResource() const
return <maxDecodingTokens, maxPathLen, maxDraftTokens, maxDraftPathLen>
-
-
-
-
-bool isLE(LookaheadDecodingConfig const &that) const
return true when
-thiscan be executed on resources defined bythat
-Public Static Functions
--
-
-
-static bool isLegal(SizeType32 windowSize, SizeType32 ngramSize, SizeType32 verificationSetSize) noexcept
return true when the parameter combination is valid.
-
Private Members
-
-
-
-SizeType32 mWindowSize
-
-
-
-SizeType32 mNgramSize
- +GuideType mGuideType
+The guide type. See GuideType.
+-
-
-
-SizeType32 mVerificationSetSize
- +std::optional<std::string> mGuide
+The detailed guide string. It could be a json schema, a regular expression or a EBNF grammar depending on mGuideType.
+@@ -4185,195 +4378,215 @@executor.h -
- -struct EagleConfig
--Public Functions
+-
+
-
+class JsonSerialization
- +#include <executor.h>+
Class with utility functions to serialize statistics to json string.
+++Public Static Functions
-
-
-
-explicit EagleConfig(std::optional<EagleChoices> eagleChoices = std::nullopt, bool greedySampling = true, std::optional<float> posteriorThreshold = std::nullopt)
- +static std::string toJsonStr(IterationStats const &iterationStats)
+Utility function to convert an iterationStats struct to a json serialized string.
+-
-
-
-bool operator==(EagleConfig const &other) const
- +static std::string toJsonStr(RequestStatsPerIteration const &requestStatsPerIter)
+Utility function to convert a requestStatsPerIteration struct to a json serialized string.
+-
-
-
-std::optional<EagleChoices> getEagleChoices() const
- +static std::string toJsonStr(RequestStats const &requestStats)
+Utility function to convert a requestStats struct to a json serialized string.
+ -
-explicit EagleConfig(std::optional<EagleChoices> eagleChoices = std::nullopt, bool greedySampling = true, std::optional<float> posteriorThreshold = std::nullopt)
-
+
-
+class KvCacheConfig
- +#include <executor.h>+
Configuration class for the KV cache.
++-Public Functions
-
-
-
-std::optional<float> getPosteriorThreshold() const
-
+explicit KvCacheConfig(bool enableBlockReuse = true, std::optional<SizeType32> const &maxTokens = std::nullopt, std::optional<std::vector<SizeType32>> const &maxAttentionWindowVec = std::nullopt, std::optional<SizeType32> const &sinkTokenLength = std::nullopt, std::optional<FloatType> const &freeGpuMemoryFraction = std::nullopt, std::optional<size_t> const &hostCacheSize = std::nullopt, bool onboardBlocks = true, std::optional<FloatType> const &crossKvCacheFraction = std::nullopt, std::optional<RetentionPriority> secondaryOffloadMinPriority = std::nullopt, size_t eventBufferMaxSize = 0, std::optional<tensorrt_llm::runtime::RuntimeDefaults> const &runtimeDefaults = std::nullopt)
--Private Functions
-
-
-
-std::optional<float> const &checkPosteriorValue(std::optional<float> const &value)
-
+std::optional<SizeType32> getMaxTokens() const
--Private Members
--
-
-
-std::optional<EagleChoices> mEagleChoices
choices forming tree for EAGLE-1.
-
-
-
-
-bool mGreedySampling
Flag to use greedy or typical acceptance.
-
-
-
-
-std::optional<float> mPosteriorThreshold
Minimum token probability of the typical acceptance. Corresponds to epsilon in https://arxiv.org/pdf/2401.10774. Default is 0.09f.
-
-
+
-
+std::optional<std::vector<SizeType32>> getMaxAttentionWindowVec() const
--Friends
--
-
- -friend class Serialization -
-
+
-
+std::optional<SizeType32> getSinkTokenLength() const
-
-std::optional<float> getPosteriorThreshold() const
-
-
-
-class ContextPhaseParams
- --
Public Types
---Public Functions
-
-
-
-explicit ContextPhaseParams(VecTokens firstGenTokens, RequestIdType reqId)
-
+std::optional<size_t> getHostCacheSize() const
-
-
-
-ContextPhaseParams(VecTokens firstGenTokens, RequestIdType reqId, void *state)
-
+bool getOnboardBlocks() const
-
-
-
-ContextPhaseParams(ContextPhaseParams const&)
-
+std::optional<RetentionPriority> getSecondaryOffloadMinPriority() const
-
-
-
-ContextPhaseParams(ContextPhaseParams&&) noexcept
-
+size_t getEventBufferMaxSize() const
-
-
-
-ContextPhaseParams &operator=(ContextPhaseParams const&)
-
+void setEnableBlockReuse(bool enableBlockReuse)
-
-
-
-ContextPhaseParams &operator=(ContextPhaseParams&&) noexcept
-
+void setMaxTokens(SizeType32 maxTokens)
-
-
-
-~ContextPhaseParams()
-
+void setMaxAttentionWindowVec(std::vector<SizeType32> maxAttentionWindowVec)
-
-
-
-bool operator==(ContextPhaseParams const&) const noexcept
-
+void setSinkTokenLength(SizeType32 sinkTokenLength)
-
-
-
-VecTokens const &getFirstGenTokens() const & noexcept
-
+void setFreeGpuMemoryFraction(FloatType freeGpuMemoryFraction)
-
-
-
-VecTokens popFirstGenTokens() && noexcept
-
+void setCrossKvCacheFraction(FloatType crossKvCacheFraction)
-
-
-
-RequestIdType getReqId() const noexcept
-
+void setHostCacheSize(size_t hostCacheSize)
-
-
-
-void *getState() noexcept
-
+void setSecondaryOffloadMinPriority(std::optional<RetentionPriority> secondaryOffloadMinPriority)
-Private Types
--
-
-
-using StatePtr = std::unique_ptr<void, decltype(&deleter)>
-
+void fillEmptyFieldsFromRuntimeDefaults(tensorrt_llm::runtime::RuntimeDefaults runtimeDefaults)
-
+
-Private Members
-
-
-
-RequestIdType mReqId = {0}
This request corresponds to the request ID in the context phase.
+-
+bool mEnableBlockReuse
Controls if KV cache blocks can be reused for different requests.
-
-
-
-VecTokens mFirstGenTokens
The first tokens generated by context executor.
+-
+std::optional<SizeType32> mMaxTokens
The maximum number of tokens that should be stored in the KV cache If both mMaxTokens and mFreeGpuMemoryFraction are specified, memory corresponding to the minimum will be allocated.
-
-
-
-StatePtr mState = {nullptr, deleter}
Context phase state of this request.
+-
+std::optional<std::vector<SizeType32>> mMaxAttentionWindowVec
Size of the attention window for each sequence. Only the last mMaxAttentionWindow tokens of each sequence will be stored in the KV cache. Different layers may have different max attention window sizes. If the number of elements in mMaxAttentionWindowVec is less than the number of layers, mMaxAttentionWindowVec will be repeated multiple times to the number of layers.
-Private Static Functions
--
-
-
-static void deleter(void const *data)
-
+
-
+std::optional<SizeType32> mSinkTokenLength
Number of sink tokens (tokens to always keep in attention window)
+
-
+
-
+std::optional<FloatType> mFreeGpuMemoryFraction
The fraction of GPU memory fraction that should be allocated for the KV cache. Default is 90%. If both mMaxTokens and mFreeGpuMemoryFraction are specified, memory corresponding to the minimum will be allocated.
+
-
+
-
+std::optional<FloatType> mCrossKvCacheFraction
The fraction of the KV Cache memory should be reserved for cross attention If set to p, self attention will use 1-p of KV Cache memory and cross attention will use p of KV Cache memory. Default is 50%. Should only be set when using encoder-decoder model.
+
-
+
-
+std::optional<size_t> mHostCacheSize
Size of secondary memory pool in bytes. Default is 0. Having a secondary memory pool increases KV cache block reuse potential.
+
-
+
-
+bool mOnboardBlocks
Controls whether offloaded blocks should be onboarded back into primary memory before being reused.
+
-
+
-
+std::optional<RetentionPriority> mSecondaryOffloadMinPriority
Only blocks with priority > mSecondaryOfflineMinPriority can be offloaded to secondary memory.
+
-
+
-
+size_t mEventBufferMaxSize
Max size of the KV cache event buffer.
+
@@ -4386,145 +4599,111 @@executor.h -
- -class SpeculativeDecodingConfig
--#include <executor.h>-Configuration for speculative decoding (both draft and target models)
---Public Functions
--
-
-
-explicit SpeculativeDecodingConfig(bool fastLogits = false)
-
-
-
-bool operator==(SpeculativeDecodingConfig const &other) const
+-
+
-
+struct KVCacheCreatedData
Public Members
-
-
-
-bool fastLogits
Send logits tensor directly from draft to target model.
+-
+std::vector<SizeType32> numBlocksPerCacheLevel
The amount of blocks at each cache level.
-
-bool fastLogits
-
-
-
-class GuidedDecodingParams
- -#include <executor.h>-
Guided decoding parameters for a request.
--Public Types
--
-
-
-enum class GuideType
Values:
--
-
-
-enumerator kJSON
The generated text is amenable to json format.
-
-
-
-
-enumerator kJSON_SCHEMA
The generated text is amenable to json format with additional user-specified restrictions, namely schema.
-
-
-
-
-enumerator kREGEX
The generated text is amenable to the user-specified regular expression.
-
-
+
-
+struct KVCacheEvent
- ++
Public Functions
+-
+
-
+KVCacheEvent(IdType eventId, KVCacheEventData data)
-
-
-
-enumerator kEBNF_GRAMMAR
The generated text is amenable to the user-specified extended Backus-Naur form (EBNF) grammar. EBNF grammar is widely-used to express context-free grammars.
+
+-Public Members
+ +-
+
-
+KVCacheEventData data
The data corresponding to this event.
-Public Functions
--
-
-
-explicit GuidedDecodingParams(GuideType guideType, std::optional<std::string> guide = std::nullopt)
-
-
-
-bool operator==(GuidedDecodingParams const &other) const
-
-
-
-GuideType getGuideType() const
-
+template<typename T>
struct KVCacheEventDiff
+ -
---
Private Members
- --
-
-
-std::optional<std::string> mGuide
The detailed guide string. It could be a json schema, a regular expression or a EBNF grammar depending on mGuideType.
+-
+
-
+class KVCacheEventManager
- +#include <executor.h>+
Exposes a limited set of KV cache manager functionalities.
++Public Functions
+-
+
+
-
+
-
+std::deque<KVCacheEvent> getLatestEvents(std::optional<std::chrono::milliseconds> timeout = std::nullopt)
Get the latest KV Cache events.
+-
+
- Parameters: +
timeout – The maximum time to wait for new events. If nullopt, will only return when new events are available, or when the executor instance has shutdown.
+
+
-Friends
--
-
- -friend class Serialization -
Private Members
+-
+
-
+std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager> kvCacheManager
-
+std::deque<KVCacheEvent> getLatestEvents(std::optional<std::chrono::milliseconds> timeout = std::nullopt)
-
-
-
-struct RetentionPriorityAndDuration
-
+struct KVCacheRemovedData
- --
Public Functions
--
-
-
-inline RetentionPriorityAndDuration(std::optional<RetentionPriority> const &retentionPriority, std::optional<std::chrono::milliseconds> const &durationMs)
Public Members
-
-
-
-std::optional<RetentionPriority> retentionPriority
-
-
-
-std::optional<std::chrono::milliseconds> durationMs
- +std::vector<IdType> blockHashes
+The hashes of blocks being removed.
+ -
-inline RetentionPriorityAndDuration(std::optional<RetentionPriority> const &retentionPriority, std::optional<std::chrono::milliseconds> const &durationMs)
executor.h
- -std::vector<TokenRangeRetentionConfig> getTokenRangeRetentionConfigs() const
+std::vector<TokenRangeRetentionConfig> getTokenRangeRetentionConfigs() const
-
+class KVCacheEventManager
-
@@ -4559,12 +4738,12 @@
-
-std::optional<std::chrono::milliseconds> getDecodeDurationMs() const
executor.h
-
-std::vector<RetentionPriorityAndDuration> getPerBlockRetentionPriorityDuration(SizeType32 blockSize, SizeType32 seqLen) const
Convert the token range data into an entry per kv block. Returns a tuple of vectors corresponding to the priorities and durations for each block.
executor.h
- RetentionPriority priority
-The priority of this token range. Higher priorities are less likely to be evicted or offloaded.
--
-
-
-std::optional<std::chrono::milliseconds> durationMs
The duration in ms that the block should remain at the given priority level. Set to std::nullopt to have no expiration time, and keep the block at the given priority level until it gets reclaimed. After the duration has passed, the block will be moved back to the
-kDefaultRetentionPrioritylevel.
-
-std::optional<std::string> mGuide
-
+
-
-
-
-class Request
- -#include <executor.h>-
A class that holds information about the request.
--+Public Functions
--
-
-
-Request(VecTokens inputTokenIds, SizeType32 maxTokens, bool streaming = false, SamplingConfig const &samplingConfig = SamplingConfig(), OutputConfig const &outputConfig = OutputConfig(), std::optional<SizeType32> const &endId = std::nullopt, std::optional<SizeType32> const &padId = std::nullopt, std::optional<std::vector<SizeType32>> positionIds = std::nullopt, std::optional<std::list<VecTokens>> badWords = std::nullopt, std::optional<std::list<VecTokens>> stopWords = std::nullopt, std::optional<Tensor> embeddingBias = std::nullopt, std::optional<ExternalDraftTokensConfig> externalDraftTokensConfig = std::nullopt, std::optional<PromptTuningConfig> pTuningConfig = std::nullopt, std::optional<MropeConfig> mRopeConfig = std::nullopt, std::optional<LoraConfig> loraConfig = std::nullopt, std::optional<LookaheadDecodingConfig> lookaheadConfig = std::nullopt, std::optional<KvCacheRetentionConfig> kvCacheRetentionConfig = std::nullopt, std::optional<std::string> logitsPostProcessorName = std::nullopt, std::optional<VecTokens> encoderInputTokenIds = std::nullopt, std::optional<IdType> clientId = std::nullopt, bool returnAllGeneratedTokens = false, PriorityType priority = kDefaultPriority, RequestType type = RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION, std::optional<ContextPhaseParams> contextPhaseParams = std::nullopt, std::optional<Tensor> encoderInputFeatures = std::nullopt, std::optional<SizeType32> encoderOutputLength = std::nullopt, std::optional<Tensor> crossAttentionMask = std::nullopt, SizeType32 numReturnSequences = 1, std::optional<EagleConfig> eagleConfig = std::nullopt, std::optional<Tensor> skipCrossAttnBlocks = std::nullopt, std::optional<GuidedDecodingParams> guidedDecodingParams = std::nullopt, std::optional<MillisecondsType> allottedTimeMs = std::nullopt)
The Request constructor.
--
-
- Parameters: -
-
-
inputTokenIds – The input token ids
-maxTokens – The maximum number of tokens to generate
-streaming – Indicates if the responses should be streamed or not. Default is false.
-samplingConfig – The sampling configuration
-outputConfig – The output configuration
-endId – The end token id
-padId – The pad token id
-positionIds – The input position ids
-badWords – A list of bad words tokens. Each “word” can be composed of multiple tokens
-stopWords – A list of stop words tokens. Each “word” can be composed of multiple tokens
-embeddingBias – The embedding bias tensor. Expected type is kFP32 and shape is [vocab_size]
-externalDraftTokensConfig – The speculative decoding with external draft tokens configuration
-pTuningConfig – The prompt tuning configuration
-loraConfig – The LoRA configuration
-lookaheadConfig – The lookahead speculative decoding configuration
-logitsPostProcessorName – The logits postprocessor name. Must correspond to one of the logits postprocessor
-kvCacheRetentionConfig – The configuration used for KV cache block eviction. name provided to the ExecutorConfig.
-encoderInputTokenIds – The encoder input token ids for encoder-decoder models, or encoder-only models
-returnAllGeneratedTokens – Indicates whether to return the full beams or just the newly generated tokens after every streaming step.
-priority – Sets the execution priority of this request.
-encoderInputFeatures – Encoder input features for multimodal models.
-encoderOutputLength – Encoder output length if encoder input and output have different lengths (due to convolution down-sampling, etc.)
-crossAttentionMask – Cross attention mask.
-type – Indicate the request type for disaggregated serving mode.
-contextPhaseParams – Generated token ID from context only executor.
-numReturnSequences – The number of returning sequences.
-eagleConfig – The EAGLE speculative decoding configuration
-skipCrossAttnBlocks – Skip the cross attention transformer blocks or not.
-guidedDecodingParams – The guided decoding parameters.
-allottedTimeMs – The allotted time in milliseconds after which the request is finished with a timedOut finish reason. The request always will exceed this time slightly, but at most with 1 forward pass. A request can be timed-out before ever being scheduled.
-
-
-
-
-
-~Request()
-
-
-
-SizeType32 getMaxTokens() const
-
-
-
-SizeType32 getMaxNewTokens() const
-
-
-
-bool getStreaming() const
-
-
-
-SamplingConfig getSamplingConfig() const
-
-
-
-OutputConfig getOutputConfig() const
-
-
-
-std::optional<SizeType32> getEndId() const
-
-
-
-std::optional<SizeType32> getPadId() const
-
-
-
-std::optional<std::vector<SizeType32>> getPositionIds() const
-
-
-
-std::optional<ExternalDraftTokensConfig> getExternalDraftTokensConfig() const
-
-
-
-std::optional<PromptTuningConfig> getPromptTuningConfig() const
-
-
-
-std::optional<MropeConfig> getMropeConfig() const
-
-
-
-std::optional<LoraConfig> getLoraConfig() const
-
-
-
-std::optional<LookaheadDecodingConfig> getLookaheadConfig() const
-
-
-
-std::optional<KvCacheRetentionConfig> getKvCacheRetentionConfig() const
-
-
-
-std::optional<std::string> getLogitsPostProcessorName() const
-
-
-
-PriorityType getPriority() const
-
-
-
-bool getReturnAllGeneratedTokens() const
The priority of this token range. Higher priorities are less likely to be evicted or offloaded.
+-
-
-
-std::optional<ContextPhaseParams> const &getContextPhaseParams() const
-
+
-
+std::optional<std::chrono::milliseconds> durationMs
The duration in ms that the block should remain at the given priority level. Set to std::nullopt to have no expiration time, and keep the block at the given priority level until it gets reclaimed. After the duration has passed, the block will be moved back to the
+kDefaultRetentionPrioritylevel.
-
-Request(VecTokens inputTokenIds, SizeType32 maxTokens, bool streaming = false, SamplingConfig const &samplingConfig = SamplingConfig(), OutputConfig const &outputConfig = OutputConfig(), std::optional<SizeType32> const &endId = std::nullopt, std::optional<SizeType32> const &padId = std::nullopt, std::optional<std::vector<SizeType32>> positionIds = std::nullopt, std::optional<std::list<VecTokens>> badWords = std::nullopt, std::optional<std::list<VecTokens>> stopWords = std::nullopt, std::optional<Tensor> embeddingBias = std::nullopt, std::optional<ExternalDraftTokensConfig> externalDraftTokensConfig = std::nullopt, std::optional<PromptTuningConfig> pTuningConfig = std::nullopt, std::optional<MropeConfig> mRopeConfig = std::nullopt, std::optional<LoraConfig> loraConfig = std::nullopt, std::optional<LookaheadDecodingConfig> lookaheadConfig = std::nullopt, std::optional<KvCacheRetentionConfig> kvCacheRetentionConfig = std::nullopt, std::optional<std::string> logitsPostProcessorName = std::nullopt, std::optional<VecTokens> encoderInputTokenIds = std::nullopt, std::optional<IdType> clientId = std::nullopt, bool returnAllGeneratedTokens = false, PriorityType priority = kDefaultPriority, RequestType type = RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION, std::optional<ContextPhaseParams> contextPhaseParams = std::nullopt, std::optional<Tensor> encoderInputFeatures = std::nullopt, std::optional<SizeType32> encoderOutputLength = std::nullopt, std::optional<Tensor> crossAttentionMask = std::nullopt, SizeType32 numReturnSequences = 1, std::optional<EagleConfig> eagleConfig = std::nullopt, std::optional<Tensor> skipCrossAttnBlocks = std::nullopt, std::optional<GuidedDecodingParams> guidedDecodingParams = std::nullopt, std::optional<MillisecondsType> allottedTimeMs = std::nullopt)
-
-
-
-std::optional<SizeType32> getEncoderOutputLength() const
-
+
-
+struct KVCacheStoredBlockData
- +#include <executor.h>+
An entry for a single block stored into the tree.
+++Public Functions
-
-
-
-std::optional<Tensor> getCrossAttentionMask() const
-
+inline KVCacheStoredBlockData(IdType blockHash, tensorrt_llm::runtime::VecUniqueTokens tokens, tensorrt_llm::runtime::LoraTaskIdType loraId, SizeType32 cacheLevel, SizeType32 priority)
-
-
-
-RequestType getRequestType() const
++Public Members
+ --
-
-
-SizeType32 getNumReturnSequences() const
-
+
-
+tensorrt_llm::runtime::VecUniqueTokens tokens
The unique tokens of the block.
+
-
-
-
-std::optional<EagleConfig> getEagleConfig() const
-
+
-
+tensorrt_llm::runtime::LoraTaskIdType loraId
The Lora task id of the block.
+
-
+
-
+SizeType32 cacheLevel
The cache level of the block.
+
-
-
-
-std::optional<GuidedDecodingParams> getGuidedDecodingParams() const
-
+
-
+SizeType32 priority
The priority of the block.
+
-
-
-
-std::optional<MillisecondsType> getAllottedTimeMs() const
-
-std::optional<Tensor> getCrossAttentionMask() const
-
-
-
-void setStreaming(bool streaming)
-
+
-
+struct KVCacheStoredData
- ++
Public Members
+ --
-
-
-void setSamplingConfig(SamplingConfig const &config)
-
+
-
+std::vector<KVCacheStoredBlockData> blocks
A sequence of blocks. The parent of block
+iis blocki-1
-
-
-
-void setOutputConfig(OutputConfig const &outputConfig)
-
-void setSamplingConfig(SamplingConfig const &config)
-
+
-
+struct KVCacheUpdatedData
- ++
Public Functions
-
-
-
-void setEndId(SizeType32 endId)
-
+inline explicit KVCacheUpdatedData(IdType blockHash)
-
-
-
-void setPadId(SizeType32 padId)
-
+inline KVCacheUpdatedData &cacheLevelUpdated(SizeType32 oldValue, SizeType32 newValue)
-
-
-
-void setPositionIds(std::vector<SizeType32> const &positionIds)
-
+inline KVCacheUpdatedData &priorityUpdated(SizeType32 oldValue, SizeType32 newValue)
++Public Members
+ - +-
+
-
+std::optional<KVCacheEventDiff<SizeType32>> cacheLevel = std::nullopt
The updated value of the cacheLevel field.
+
-
+
-
+std::optional<KVCacheEventDiff<SizeType32>> priority = std::nullopt
The updated value of the priority field.
+
-
-
-
-void setExternalDraftTokensConfig(ExternalDraftTokensConfig const &externalDraftTokensConfig)
-
-void setEndId(SizeType32 endId)
-
+
-
+class LogitsPostProcessorConfig
- ++
Public Functions
-
-
-
-void setPromptTuningConfig(PromptTuningConfig const &pTuningConfig)
-
+explicit LogitsPostProcessorConfig(std::optional<LogitsPostProcessorMap> processorMap = std::nullopt, std::optional<LogitsPostProcessorBatched> processorBatched = std::nullopt, bool replicate = true)
-
-
-
-void setMropeConfig(MropeConfig const &mRopeConfig)
-
+std::optional<LogitsPostProcessorMap> getProcessorMap() const
-
-
-
-void setLoraConfig(LoraConfig const &loraConfig)
-
+std::optional<LogitsPostProcessorBatched> getProcessorBatched() const
-
-
-
-void setLookaheadConfig(LookaheadDecodingConfig const &lookaheadConfig)
-
+bool getReplicate() const
-
-
-
-void setKvCacheRetentionConfig(KvCacheRetentionConfig const &kvCacheRetentionConfig)
-
+void setProcessorMap(LogitsPostProcessorMap const &processorMap)
-
-
-
-void setLogitsPostProcessorName(std::string const &logitsPostProcessorName)
-
+void setProcessorBatched(LogitsPostProcessorBatched const &processorBatched)
++Private Members
+-
+
-
+std::optional<LogitsPostProcessorMap> mProcessorMap
mapping from post processor names to non-batched post processors
+
-
-
-
-void setPriority(PriorityType priority)
-
+
-
+std::optional<LogitsPostProcessorBatched> mProcessorBatched
single batched post processor
+
-
-
-
-void setReturnAllGeneratedTokens(bool returnAllGeneratedTokens)
-
+
-
+bool mReplicate
If set to true, logits post processor will run on all TP ranks in last PP rank.
+
-
-void setPromptTuningConfig(PromptTuningConfig const &pTuningConfig)
-
+
-
+struct LookaheadDecodingConfig
- ++
Public Functions
-
-
-
-void setRequestType(RequestType const &requestType)
-
+LookaheadDecodingConfig(SizeType32 windowSize, SizeType32 ngramSize, SizeType32 verificationSetSize)
-
-
-
-void setContextPhaseParams(ContextPhaseParams contextPhaseParams)
-
+inline explicit LookaheadDecodingConfig()
-
-
-
-void setEncoderInputFeatures(Tensor encoderInputFeatures)
-
+bool operator==(LookaheadDecodingConfig const &other) const
-
-
-
-void setEncoderOutputLength(SizeType32 encoderOutputLength)
-
+std::tuple<SizeType32 const, SizeType32 const, SizeType32 const> get() const
-
-
-
-void setCrossAttentionMask(Tensor crossAttentionMask)
-
+SizeType32 getWindowSize() const
-
-
-
-void setNumReturnSequences(SizeType32 numReturnSequences)
-
+SizeType32 getNgramSize() const
-
-
-
-void setEagleConfig(std::optional<EagleConfig> const &eagleConfig)
-
+SizeType32 getVerificationSetSize() const
- +std::tuple<SizeType32, SizeType32, SizeType32, SizeType32> calculateSpeculativeResource() const
+return <maxDecodingTokens, maxPathLen, maxDraftTokens, maxDraftPathLen>
+-
-
-
-void setGuidedDecodingParams(GuidedDecodingParams const &guidedDecodingParams)
- +bool isLE(LookaheadDecodingConfig const &that) const
+return true when
+thiscan be executed on resources defined bythat+Public Static Functions
-
-
-
-void setAllottedTimeMs(MillisecondsType allottedTimeMs)
- +static bool isLegal(SizeType32 windowSize, SizeType32 ngramSize, SizeType32 verificationSetSize) noexcept
+return true when the parameter combination is valid.
+-Public Static Attributes
+Private Members
-
-
-
-static constexpr PriorityType kDefaultPriority = 0.5
-
+SizeType32 mWindowSize
-
-
-
-static constexpr auto kBatchedPostProcessorName = "batched"
This logits postprocessor name will dispatch to the batched logits postprocessor.
-
- +SizeType32 mNgramSize
+ + +-
+
-
+SizeType32 mVerificationSetSize
-@@ -5081,363 +5122,387 @@Private Members
+Private Static Attributes
-
-
-
-std::unique_ptr<Impl> mImpl
-
+static constexpr SizeType32 kDefaultLookaheadDecodingWindow = 4
-
+
-
+static constexpr SizeType32 kDefaultLookaheadDecodingNgram = 3
-
+
-
+static constexpr SizeType32 kDefaultLookaheadDecodingVerificationSet = 4
executor.h -
- -struct SpeculativeDecodingFastLogitsInfo
+-
+
-
+class LoraConfig
- #include <executor.h>-
Struct that holds the logits information when using direct transfer.
+Configuration for LoRA.
- -
-
-
-
-struct Result
- -#include <executor.h>-
Struct that holds the generation result.
+
-
-void setRequestType(RequestType const &requestType)
-+Public Members
--
-
-
-bool isFinal
Indicates if this is the final result for the request.
-
-
-
-
-BeamTokens outputTokenIds
The output tokens for each beam.
-
-
-
-
-std::optional<VecLogProbs> cumLogProbs
The cumulative log probabilities. Size beamSize.
-
Private Members
-
-
-
-std::optional<std::vector<VecLogProbs>> logProbs
The log probabilities for each generated token. Size [beamSize, outputLen].
+-
+IdType mTaskId
The Lora task id.
-
-
-
-std::optional<Tensor> contextLogits
The context logits. Size [promptLen, vocabSizePadded].
+-
+std::optional<Tensor> mWeights
The Lora weights. See TRT-LLM documentation for expected shapes and types.
-
-
-
-std::optional<Tensor> generationLogits
The generation logits. Size [beamSize, maxNewTokens, vocabSizePadded] (non-streaming) or [maxNewTokens, beamSize, vocabSizePadded] (streaming and allGeneratedTokens) or [1, beamSize, vocabSizePadded] (streaming and non-allGeneratedTokens)
+-
+std::optional<Tensor> mConfig
The Lora configuration. See TRT-LLM documentation for detailed description of the config tensor.
-
-
-
-std::optional<SpeculativeDecodingFastLogitsInfo> specDecFastLogitsInfo
Logits information for direct transfer when using fast logits.
-
-
+KVCacheEvent(IdType eventId, KVCacheEventData data)
-
-
-
-std::vector<FinishReason> finishReasons
The reason why the model stopped generating tokens for each beam in this request. Size [beamSize]. Currently only supported when beamSize is 1 and when using BatchingType::kINFLIGHT.
-
-
+
-
+class MropeConfig
- +#include <executor.h>+
Configuration for mrope.
+++Public Functions
+-
+
-
+explicit MropeConfig(Tensor mropeRoratySinCos, SizeType32 mropePositionDeltas)
-
-
-
-std::optional<ContextPhaseParams> contextPhaseParams
The params of the context phase.
-
-
-
-
-SizeType32 decodingIter = {0}
The number of the decoding iterations used to generate the result. In autoregressive decoding, it is equal to the maximum length of the beam in outputTokenIds. In speculative decoding, might be less than maximum length of the beam in outputTokenIds as more than one token can be generated per iteration. Used for speculative decoding statistics.
-
-
+
-
+SizeType32 getMRopePositionDeltas() const
++Private Members
-
-
-
-SizeType32 sequenceIndex = {0}
The index of the output sequence of this result where 0 <= sequenceIndex < numReturnSequences. In beam search (beamWidth > 1), this index will be always zero because all beams to be returned are included in this result.
+-
+Tensor mMRopeRotaryCosSin
The mrope rotary sin and cos cache. Expected shape: [maxPositionEmbeddings*rotaryEmbeddingDim],Data type must float32.
-
-
-
-bool isSequenceFinal
Indicates if this is the final result for a given sequence in the request In beam search (beamWidth > 1), the value will always equal to the value of isFinal.
+-
+SizeType32 mMRopePositionDeltas
The mrope position deltas.
-
-
-
-std::optional<RequestPerfMetrics> requestPerfMetrics
Performance metrics if returnPerfMetrics is set in OutputConfig.
-
+Friends
+-
+
- +friend class Serialization +
-
+explicit MropeConfig(Tensor mropeRoratySinCos, SizeType32 mropePositionDeltas)
-
-
-
-class Response
- -#include <executor.h>-
Class that holds either an error or a result.
-+- +class OrchestratorConfig
+-Public Functions
-
-
-
-Response(IdType requestId, std::string errorMsg, std::optional<IdType> clientId = std::nullopt)
-
-
-
-std::optional<IdType> getClientId() const
Get the client id of the request for which this response was generated.
-
-
-
-
-bool hasError() const
Indicates if this response has an error or not.
-
- +void setWorkerExecutablePath(std::string const &workerExecutablePath)
+-
-
-
-std::string const &getErrorMsg() const
Get the error msg for this response Will throw an exception if hasError is false.
-
-
-
-
-Result const &getResult() const
Get the result for this response Will throw an exception if hasResult is true.
-
- +void setSpawnProcesses(bool spawnProcesses)
+-
-
-
-class DynamicBatchConfig
-
+class OutputConfig
- #include <executor.h>-
Configuration class for dynamic tuning of batch size and max num tokens. During runtime the statistics of input and output lengths are recoreded. Based on these statistics, the batch size and max num tokens are tuned dynamically to better serve the requests.
+Configuration that controls the outputs of a Result.
Public Functions
-
-
-
-explicit DynamicBatchConfig(bool enableBatchSizeTuning = false, bool enableMaxNumTokensTuning = false, SizeType32 dynamicBatchMovingAverageWindow = kDefaultDynamicBatchMovingAverageWindow, std::vector<std::pair<SizeType32, SizeType32>> batchSizeTable = kDefaultBatchSizeTable)
-
-
-
-SizeType32 getDynamicBatchMovingAverageWindow() const
-
-
-
-bool getEnableBatchSizeTuning() const
-
-
-
-bool getEnableMaxNumTokensTuning() const
-
-
-
-std::vector<std::pair<SizeType32, SizeType32>> getBatchSizeTable() const
-
+explicit OutputConfig(bool returnLogProbs = false, bool returnContextLogits = false, bool returnGenerationLogits = false, bool excludeInputFromOutput = false, bool returnEncoderOutput = false, bool returnPerfMetrics = false, std::optional<std::vector<AdditionalModelOutput>> additionalModelOutputs = std::nullopt)
--Public Static Attributes
+Public Members
-
-
-
-static SizeType32 const kDefaultDynamicBatchMovingAverageWindow = 128
The default window size for moving average of input and output length which is used to calculate dynamic batch size and max num tokens.
+-
+bool returnLogProbs
Controls if Result should contain log probabilities. Default is false.
-
-
-
-static std::vector<std::pair<SizeType32, SizeType32>> const kDefaultBatchSizeTable
The default value of batch size table.
+-
+bool returnContextLogits
Controls if Result should contain the context logits. Default is false.
-+Private Members
-
-
-
-bool mEnableBatchSizeTuning
Controls if the batch size should be tuned dynamically.
+-
+bool returnGenerationLogits
Controls if Result should contain the generation logits. Default is false.
-
-
-
-bool mEnableMaxNumTokensTuning
Controls if the max num tokens should be tuned dynamically.
+-
+bool excludeInputFromOutput
Controls if output tokens in Result should include the input tokens. Default is false.
-
-
-
-SizeType32 mDynamicBatchMovingAverageWindow
The window size for moving average of input and output length which is used to calculate dynamic batch size and max num tokens.
+-
+bool returnEncoderOutput
Controls if Result should contain encoder output hidden states (for encoder-only and encoder-decoder models). Default is false.
-
-
-
-std::vector<std::pair<SizeType32, SizeType32>> mBatchSizeTable
A vector of (batchSizeLimit, batchSize). When max capacity batch size is less than.
+-
+bool returnPerfMetrics
Controls if Result should contain performance metrics.
+
-
+
-
+std::optional<std::vector<AdditionalModelOutput>> additionalModelOutputs
The additional outputs to gather from the model.
-
+
-
+class AdditionalModelOutput
- +#include <executor.h>+
Additional output that should be gathered.
+By default gather output of shape [beamWidth, x] from each generation phase. If gatherContext is true, also gather output of shape [promptLen, x] from context phase.
-+Friends
--
-
- -friend class Serialization -
Public Functions
+-
+
-
+explicit AdditionalModelOutput(std::string name, bool gatherContext = false)
-
-explicit DynamicBatchConfig(bool enableBatchSizeTuning = false, bool enableMaxNumTokensTuning = false, SizeType32 dynamicBatchMovingAverageWindow = kDefaultDynamicBatchMovingAverageWindow, std::vector<std::pair<SizeType32, SizeType32>> batchSizeTable = kDefaultBatchSizeTable)
-
-
-
-class SchedulerConfig
-
+class ParallelConfig
- #include <executor.h>-
Configuration class for the scheduler.
+A configuration class for the parallel execution parameters Currently only supports commType = CommunicationType::kMPI.
Public Functions
-
-
-
-explicit SchedulerConfig(CapacitySchedulerPolicy capacitySchedulerPolicy = CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT, std::optional<ContextChunkingPolicy> contextChunkingPolicy = std::nullopt, std::optional<DynamicBatchConfig> dynamicBatchConfig = std::nullopt)
-
+explicit ParallelConfig(CommunicationType commType = CommunicationType::kMPI, CommunicationMode commMode = CommunicationMode::kLEADER, std::optional<std::vector<SizeType32>> deviceIds = std::nullopt, std::optional<std::vector<SizeType32>> participantIds = std::nullopt, std::optional<OrchestratorConfig> const &orchestratorConfig = std::nullopt)
Constructor.
+-
+
- Parameters: +
-
+
commType – The communication type. See CommunicationType.
+commMode – The communication mode. See CommunicationMode.
+deviceIds – The IDs of the GPUs involved in the execution of the model
+participantIds – The participant IDs (MPI ranks if commType == kMPI) involved in the execution of the model. The first participant is considered to be the leader.
+
+
-
+
-
+CommunicationType getCommunicationType() const
-
-
-
-bool operator==(SchedulerConfig const &other) const
-
+CommunicationMode getCommunicationMode() const
-
-
-
-CapacitySchedulerPolicy getCapacitySchedulerPolicy() const
-
+std::optional<std::vector<SizeType32>> getDeviceIds() const
-
-
-
-std::optional<ContextChunkingPolicy> getContextChunkingPolicy() const
-
+std::optional<std::vector<SizeType32>> getParticipantIds() const
-
-
-
-std::optional<DynamicBatchConfig> getDynamicBatchConfig() const
-
+std::optional<OrchestratorConfig> getOrchestratorConfig() const
-
+
-
+void setCommunicationType(CommunicationType type)
-
+
-
+void setCommunicationMode(CommunicationMode mode)
-
+
-
+void setDeviceIds(std::vector<SizeType32> const &deviceIds)
-
+
-
+void setParticipantIds(std::vector<SizeType32> const &participantIds)
-
+
-
+void setOrchestratorConfig(OrchestratorConfig const &orchestratorConfig)
@@ -5452,185 +5517,158 @@Private Members
-
-
-
-CapacitySchedulerPolicy mCapacitySchedulerPolicy
The capacity scheduler policy. See CapacitySchedulerPolicy.
+-
+CommunicationType mCommType
The type of communication protocol used. Default is MPI.
-
-
-
-std::optional<ContextChunkingPolicy> mContextChunkingPolicy
The context chunking policy. See ContextChunkingPolicy.
+-
+CommunicationMode mCommMode
The mode of communication. See CommunicationMode.
-
-
-
-std::optional<DynamicBatchConfig> mDynamicBatchConfig
The config for tuning batch size dynamically. See DynamicBatchSizeConfig.
+-
+std::optional<std::vector<SizeType32>> mDeviceIds
The GPU device ids to use for executing this model.
+
-
+
-
+std::optional<std::vector<SizeType32>> mParticipantIds
The participant ids (MPI ranks for example) used for executing this model.
+
-
+
-
+std::optional<OrchestratorConfig> mOrchestratorConfig
Optional orchestrator configuration.
executor.h -
- -class KvCacheConfig
+- +class PeftCacheConfig
#include <executor.h>-Configuration class for the KV cache.
--+Public Functions
--
-
-
-explicit KvCacheConfig(bool enableBlockReuse = false, std::optional<SizeType32> const &maxTokens = std::nullopt, std::optional<std::vector<SizeType32>> const &maxAttentionWindowVec = std::nullopt, std::optional<SizeType32> const &sinkTokenLength = std::nullopt, std::optional<FloatType> const &freeGpuMemoryFraction = std::nullopt, std::optional<size_t> const &hostCacheSize = std::nullopt, bool onboardBlocks = true, std::optional<FloatType> const &crossKvCacheFraction = std::nullopt, std::optional<RetentionPriority> secondaryOffloadMinPriority = std::nullopt, size_t eventBufferMaxSize = 0, std::optional<tensorrt_llm::runtime::RuntimeDefaults> const &runtimeDefaults = std::nullopt)
-
-
-
-bool getEnableBlockReuse() const
-
-
-
-std::optional<SizeType32> getMaxTokens() const
-
-
-
-std::optional<std::vector<SizeType32>> getMaxAttentionWindowVec() const
-
-
-
-std::optional<SizeType32> getSinkTokenLength() const
config for PeftCacheManager
+++Public Functions
-
-
-
-std::optional<FloatType> getFreeGpuMemoryFraction() const
-
+explicit PeftCacheConfig(SizeType32 numHostModuleLayer = 0, SizeType32 numDeviceModuleLayer = 0, SizeType32 optimalAdapterSize = kDefaultOptimalAdapterSize, SizeType32 maxAdapterSize = kDefaultMaxAdapterSize, SizeType32 numPutWorkers = 1, SizeType32 numEnsureWorkers = 1, SizeType32 numCopyStreams = 1, SizeType32 maxPagesPerBlockHost = kDefaultMaxPagesPerBlockHost, SizeType32 maxPagesPerBlockDevice = kDefaultMaxPagesPerBlockDevice, std::optional<float> const &deviceCachePercent = std::nullopt, std::optional<size_t> const &hostCacheSize = std::nullopt)
-
-
-
-std::optional<FloatType> getCrossKvCacheFraction() const
-
+bool operator==(PeftCacheConfig const &other) const
-
-
-
-std::optional<size_t> getHostCacheSize() const
-
+SizeType32 getNumHostModuleLayer() const
-
-
-
-bool getOnboardBlocks() const
-
+SizeType32 getNumDeviceModuleLayer() const
-
-
-
-std::optional<RetentionPriority> getSecondaryOffloadMinPriority() const
-
+SizeType32 getOptimalAdapterSize() const
-
-
-
-size_t getEventBufferMaxSize() const
-
+SizeType32 getMaxAdapterSize() const
-
-
-
-void setEnableBlockReuse(bool enableBlockReuse)
-
+SizeType32 getNumPutWorkers() const
-
-
-
-void setMaxTokens(SizeType32 maxTokens)
-
+SizeType32 getNumEnsureWorkers() const
-
-
-
-void setMaxAttentionWindowVec(std::vector<SizeType32> maxAttentionWindowVec)
-
+SizeType32 getNumCopyStreams() const
-
-
-
-void setSinkTokenLength(SizeType32 sinkTokenLength)
-
+SizeType32 getMaxPagesPerBlockHost() const
-
-
-
-void setFreeGpuMemoryFraction(FloatType freeGpuMemoryFraction)
-
+SizeType32 getMaxPagesPerBlockDevice() const
-
-
-
-void setCrossKvCacheFraction(FloatType crossKvCacheFraction)
-
+std::optional<float> getDeviceCachePercent() const
-
-
-
-void setOnboardBlocks(bool onboardBlocks)
+Public Static Attributes
+-
+
-
+static constexpr SizeType32 kDefaultOptimalAdapterSize = 8
-
-
-
-void setSecondaryOffloadMinPriority(std::optional<RetentionPriority> secondaryOffloadMinPriority)
-
+static constexpr SizeType32 kDefaultMaxAdapterSize = 64
-
-void setEventBufferMaxSize(size_t eventBufferMaxSize)
-
+static constexpr SizeType32 kDefaultMaxPagesPerBlockHost = 24
-
-void fillEmptyFieldsFromRuntimeDefaults(tensorrt_llm::runtime::RuntimeDefaults runtimeDefaults)
-
+static constexpr SizeType32 kDefaultMaxPagesPerBlockDevice = 8
-
+
-
-
-
+
-
-
-
+
Private Members
-
-
-
-bool mEnableBlockReuse
Controls if KV cache blocks can be reused for different requests.
-
- +SizeType32 mNumHostModuleLayer
+-
-
-
-std::optional<SizeType32> mMaxTokens
The maximum number of tokens that should be stored in the KV cache If both mMaxTokens and mFreeGpuMemoryFraction are specified, memory corresponding to the minimum will be allocated.
-
- +SizeType32 mNumDeviceModuleLayer
+-
-
-
-std::optional<std::vector<SizeType32>> mMaxAttentionWindowVec
Size of the attention window for each sequence. Only the last mMaxAttentionWindow tokens of each sequence will be stored in the KV cache. Different layers may have different max attention window sizes. If the number of elements in mMaxAttentionWindowVec is less than the number of layers, mMaxAttentionWindowVec will be repeated multiple times to the number of layers.
-
- +SizeType32 mOptimalAdapterSize
+-
-
-
-std::optional<SizeType32> mSinkTokenLength
Number of sink tokens (tokens to always keep in attention window)
-
- +SizeType32 mMaxAdapterSize
+-
-
-
-std::optional<FloatType> mFreeGpuMemoryFraction
The fraction of GPU memory fraction that should be allocated for the KV cache. Default is 90%. If both mMaxTokens and mFreeGpuMemoryFraction are specified, memory corresponding to the minimum will be allocated.
-
- +SizeType32 mNumPutWorkers
+-
-
-
-std::optional<FloatType> mCrossKvCacheFraction
The fraction of the KV Cache memory should be reserved for cross attention If set to p, self attention will use 1-p of KV Cache memory and cross attention will use p of KV Cache memory. Default is 50%. Should only be set when using encoder-decoder model.
-
- +SizeType32 mNumEnsureWorkers
+-
-
-
-std::optional<size_t> mHostCacheSize
Size of secondary memory pool in bytes. Default is 0. Having a secondary memory pool increases KV cache block reuse potential.
-
- +SizeType32 mNumCopyStreams
+-
-
-
-bool mOnboardBlocks
Controls whether offloaded blocks should be onboarded back into primary memory before being reused.
-
- +SizeType32 mMaxPagesPerBlockHost
+-
-
-
-std::optional<RetentionPriority> mSecondaryOffloadMinPriority
Only blocks with priority > mSecondaryOfflineMinPriority can be offloaded to secondary memory.
-
- +SizeType32 mMaxPagesPerBlockDevice
+-
-
-
-size_t mEventBufferMaxSize
Max size of the KV cache event buffer.
-
- +std::optional<FloatType> mDeviceCachePercent
+ + +-
+
-
+std::optional<size_t> mHostCacheSize
@@ -5644,88 +5682,41 @@+ +executor.h -
- -class ExtendedRuntimePerfKnobConfig
+- +class PromptTuningConfig
#include <executor.h>-Configuration class for the runtime perf knobs.
+Configuration for prompt tuning.
Public Functions
-
-
-
-explicit ExtendedRuntimePerfKnobConfig(bool multiBlockMode = true, bool enableContextFMHAFP32Acc = false, bool cudaGraphMode = false, SizeType32 cudaGraphCacheSize = 0)
-
-
-
-inline bool operator==(ExtendedRuntimePerfKnobConfig const &other) const
-
-
-
-bool getMultiBlockMode() const
-
-
-
-bool getEnableContextFMHAFP32Acc() const
-
-
-
-bool getCudaGraphMode() const
-
-
-
-SizeType32 getCudaGraphCacheSize() const
-
-
-
-void setMultiBlockMode(bool multiBlockMode)
-
-
-
-void setEnableContextFMHAFP32Acc(bool enableContextFMHAFP32Acc)
-
+explicit PromptTuningConfig(Tensor embeddingTable, std::optional<VecTokenExtraIds> inputTokenExtraIds = std::nullopt)
-
-
-
-void setCudaGraphCacheSize(SizeType32 cacheSize)
-
+std::optional<VecTokenExtraIds> getInputTokenExtraIds() const
@@ -5735,460 +5726,430 @@Private Members
-
-
-
-bool mMultiBlockMode
Control if multi block mode should be enabled or not.
-
-
-
-
-bool mEnableContextFMHAFP32Acc
If enable FMHA runner FP32 accumulation.
-
-
-
-
-bool mCudaGraphMode
Control if enable cuda graph.
+-
+Tensor mEmbeddingTable
The prompt embedding table. Expected shape: [task vocab_size, hidden_size]. Data type must match model weights.
-
-
-
-SizeType32 mCudaGraphCacheSize
Number of cuda graphs to be cached in the runtime. The larger the cache, the better the perf, but more GPU memory is consumed.
+-
+std::optional<VecTokenExtraIds> mInputTokenExtraIds
The input token extra ids for KV Cache reuse when p-tuning is enabled.
executor.h friend class Serialization - -
-
-explicit SchedulerConfig(CapacitySchedulerPolicy capacitySchedulerPolicy = CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT, std::optional<ContextChunkingPolicy> contextChunkingPolicy = std::nullopt, std::optional<DynamicBatchConfig> dynamicBatchConfig = std::nullopt)
-
+
-
+class Request
- +#include <executor.h>+
A class that holds information about the request.
++Public Functions
+-
+
-
+Request(VecTokens inputTokenIds, SizeType32 maxTokens, bool streaming = false, SamplingConfig const &samplingConfig = SamplingConfig(), OutputConfig const &outputConfig = OutputConfig(), std::optional<SizeType32> const &endId = std::nullopt, std::optional<SizeType32> const &padId = std::nullopt, std::optional<std::vector<SizeType32>> positionIds = std::nullopt, std::optional<std::list<VecTokens>> badWords = std::nullopt, std::optional<std::list<VecTokens>> stopWords = std::nullopt, std::optional<Tensor> embeddingBias = std::nullopt, std::optional<ExternalDraftTokensConfig> externalDraftTokensConfig = std::nullopt, std::optional<PromptTuningConfig> pTuningConfig = std::nullopt, std::optional<MropeConfig> mRopeConfig = std::nullopt, std::optional<LoraConfig> loraConfig = std::nullopt, std::optional<LookaheadDecodingConfig> lookaheadConfig = std::nullopt, std::optional<KvCacheRetentionConfig> kvCacheRetentionConfig = std::nullopt, std::optional<std::string> logitsPostProcessorName = std::nullopt, std::optional<VecTokens> encoderInputTokenIds = std::nullopt, std::optional<IdType> clientId = std::nullopt, bool returnAllGeneratedTokens = false, PriorityType priority = kDefaultPriority, RequestType type = RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION, std::optional<ContextPhaseParams> contextPhaseParams = std::nullopt, std::optional<Tensor> encoderInputFeatures = std::nullopt, std::optional<SizeType32> encoderOutputLength = std::nullopt, std::optional<Tensor> crossAttentionMask = std::nullopt, SizeType32 numReturnSequences = 1, std::optional<EagleConfig> eagleConfig = std::nullopt, std::optional<Tensor> skipCrossAttnBlocks = std::nullopt, std::optional<GuidedDecodingParams> guidedDecodingParams = std::nullopt, std::optional<MillisecondsType> allottedTimeMs = std::nullopt)
The Request constructor.
+-
+
- Parameters: +
-
+
inputTokenIds – The input token ids
+maxTokens – The maximum number of tokens to generate
+streaming – Indicates if the responses should be streamed or not. Default is false.
+samplingConfig – The sampling configuration
+outputConfig – The output configuration
+endId – The end token id
+padId – The pad token id
+positionIds – The input position ids
+badWords – A list of bad words tokens. Each “word” can be composed of multiple tokens
+stopWords – A list of stop words tokens. Each “word” can be composed of multiple tokens
+embeddingBias – The embedding bias tensor. Expected shape is [vocab_size]
+externalDraftTokensConfig – The speculative decoding with external draft tokens configuration
+pTuningConfig – The prompt tuning configuration
+loraConfig – The LoRA configuration
+lookaheadConfig – The lookahead speculative decoding configuration
+logitsPostProcessorName – The logits postprocessor name. Must correspond to one of the logits postprocessor
+kvCacheRetentionConfig – The configuration used for KV cache block eviction. name provided to the ExecutorConfig.
+encoderInputTokenIds – The encoder input token ids for encoder-decoder models, or encoder-only models
+returnAllGeneratedTokens – Indicates whether to return the full beams or just the newly generated tokens after every streaming step.
+priority – Sets the execution priority of this request.
+encoderInputFeatures – Encoder input features for multimodal models.
+encoderOutputLength – Encoder output length if encoder input and output have different lengths (due to convolution down-sampling, etc.)
+crossAttentionMask – Cross attention mask.
+type – Indicate the request type for disaggregated serving mode.
+contextPhaseParams – Generated token ID from context only executor.
+numReturnSequences – The number of returning sequences.
+eagleConfig – The EAGLE speculative decoding configuration
+skipCrossAttnBlocks – Skip the cross attention transformer blocks or not.
+guidedDecodingParams – The guided decoding parameters.
+allottedTimeMs – The allotted time in milliseconds after which the request is finished with a timedOut finish reason. The request always will exceed this time slightly, but at most with 1 forward pass. A request can be timed-out before ever being scheduled.
+
+
-
-
-
-class DebugConfig
- -#include <executor.h>-
Configuration class for debugging output.
---Public Functions
-
-
-
-explicit DebugConfig(bool debugInputTensors = false, bool debugOutputTensors = false, StringVec debugTensorNames = {}, SizeType32 debugTensorsMaxIterations = 0)
-
+Request(Request const &other)
-
-
-
-bool operator==(DebugConfig const &other) const
-
+Request(Request &&other) noexcept
-
-
-
-SizeType32 getDebugTensorsMaxIterations() const
-
+VecTokens getInputTokenIds() const
-
-
-
-void setDebugInputTensors(bool debugInputTensors)
-
+SizeType32 getMaxTokens() const
-
-
-
-void setDebugOutputTensors(bool debugOutputTensors)
-
+SizeType32 getMaxNewTokens() const
-
-
-
-void setDebugTensorsMaxIterations(SizeType32 debugTensorsMaxIterations)
-
+SamplingConfig getSamplingConfig() const
--Private Types
--
-
-
-using StringVec = std::vector<std::string>
-
+OutputConfig getOutputConfig() const
-
+
-- -Private Members
--
-
-
-bool mDebugInputTensors
If true, debug all input tensors.
-
-
-
-
-bool mDebugOutputTensors
If true, debug all output tensors.
-
-
-
-
-SizeType32 mDebugTensorsMaxIterations
If > 0, provide debug tensors for at most debugTensorsMaxIterations past iterations, else dump them to files.
-
-
-explicit DebugConfig(bool debugInputTensors = false, bool debugOutputTensors = false, StringVec debugTensorNames = {}, SizeType32 debugTensorsMaxIterations = 0)
-
+
-
+std::optional<SizeType32> getPadId() const
-
-
-
-class OrchestratorConfig
- --
Public Functions
-
-
+
-
+std::optional<std::vector<SizeType32>> getPositionIds() const
-
-
-
-std::string getWorkerExecutablePath() const
-
+std::optional<std::list<VecTokens>> getStopWords() const
-
-
-
-std::shared_ptr<mpi::MpiComm> getOrchLeaderComm() const
-
+std::optional<Tensor> getEmbeddingBias() const
-
-
-
-bool getSpawnProcesses() const
-
+std::optional<ExternalDraftTokensConfig> getExternalDraftTokensConfig() const
-
-
-
-void setIsOrchestrator(bool isOrchestrator)
-
+std::optional<PromptTuningConfig> getPromptTuningConfig() const
-
-
-
-void setWorkerExecutablePath(std::string const &workerExecutablePath)
-
+std::optional<MropeConfig> getMropeConfig() const
-
-
+
-
+std::optional<LoraConfig> getLoraConfig() const
-
-
-
-void setSpawnProcesses(bool spawnProcesses)
-
+std::optional<LookaheadDecodingConfig> getLookaheadConfig() const
--Private Members
--
-
-
-bool mIsOrchestrator
-
+std::optional<KvCacheRetentionConfig> getKvCacheRetentionConfig() const
-
+
-
+std::optional<std::vector<SizeType32>> getPositionIds() const
-
+
-
+PriorityType getPriority() const
-
-
-
-class ParallelConfig
- -#include <executor.h>-
A configuration class for the parallel execution parameters Currently only supports commType = CommunicationType::kMPI.
---Public Functions
-
-
-
-explicit ParallelConfig(CommunicationType commType = CommunicationType::kMPI, CommunicationMode commMode = CommunicationMode::kLEADER, std::optional<std::vector<SizeType32>> deviceIds = std::nullopt, std::optional<std::vector<SizeType32>> participantIds = std::nullopt, std::optional<OrchestratorConfig> const &orchestratorConfig = std::nullopt)
Constructor.
--
-
- Parameters: -
-
-
commType – The communication type. See CommunicationType.
-commMode – The communication mode. See CommunicationMode.
-deviceIds – The IDs of the GPUs involved in the execution of the model
-participantIds – The participant IDs (MPI ranks if commType == kMPI) involved in the execution of the model. The first participant is considered to be the leader.
-
-
- +bool getReturnAllGeneratedTokens() const
+-
-
-
-CommunicationType getCommunicationType() const
-
+std::optional<ContextPhaseParams> const &getContextPhaseParams() const
-
-
-
-CommunicationMode getCommunicationMode() const
-
+std::optional<Tensor> getEncoderInputFeatures() const
-
-
-
-std::optional<std::vector<SizeType32>> getDeviceIds() const
-
+std::optional<SizeType32> getEncoderOutputLength() const
-
-
-
-std::optional<std::vector<SizeType32>> getParticipantIds() const
-
+std::optional<Tensor> getCrossAttentionMask() const
-
-
-
-std::optional<OrchestratorConfig> getOrchestratorConfig() const
-
+RequestType getRequestType() const
-
-
-
-void setCommunicationType(CommunicationType type)
-
+SizeType32 getNumReturnSequences() const
-
-
-
-void setCommunicationMode(CommunicationMode mode)
-
+std::optional<EagleConfig> getEagleConfig() const
-
-
-
-void setDeviceIds(std::vector<SizeType32> const &deviceIds)
-
+std::optional<Tensor> getSkipCrossAttnBlocks() const
-
-
-
-void setParticipantIds(std::vector<SizeType32> const &participantIds)
-
+std::optional<GuidedDecodingParams> getGuidedDecodingParams() const
-
-
-
-void setOrchestratorConfig(OrchestratorConfig const &orchestratorConfig)
-
+std::optional<MillisecondsType> getAllottedTimeMs() const
-- -Private Members
--
-
-
-CommunicationType mCommType
The type of communication protocol used. Default is MPI.
-
-
+
-
+std::optional<std::vector<std::string>> getAdditionalOutputNames() const
-
-
-
-CommunicationMode mCommMode
The mode of communication. See CommunicationMode.
-
-
+
-
+void setStreaming(bool streaming)
-
-
-
-std::optional<std::vector<SizeType32>> mDeviceIds
The GPU device ids to use for executing this model.
-
-
+
-
+void setSamplingConfig(SamplingConfig const &config)
-
-
-
-std::optional<std::vector<SizeType32>> mParticipantIds
The participant ids (MPI ranks for example) used for executing this model.
-
-
+
-
+void setOutputConfig(OutputConfig const &outputConfig)
-
-
-
-std::optional<OrchestratorConfig> mOrchestratorConfig
Optional orchestrator configuration.
-
-
+
-
+void setEndId(SizeType32 endId)
-
-explicit ParallelConfig(CommunicationType commType = CommunicationType::kMPI, CommunicationMode commMode = CommunicationMode::kLEADER, std::optional<std::vector<SizeType32>> deviceIds = std::nullopt, std::optional<std::vector<SizeType32>> participantIds = std::nullopt, std::optional<OrchestratorConfig> const &orchestratorConfig = std::nullopt)
-
+
-
+void setPositionIds(std::vector<SizeType32> const &positionIds)
-
-
-
-class PeftCacheConfig
- -#include <executor.h>-
config for PeftCacheManager
---Public Functions
-
-
-
-explicit PeftCacheConfig(SizeType32 numHostModuleLayer = 0, SizeType32 numDeviceModuleLayer = 0, SizeType32 optimalAdapterSize = kDefaultOptimalAdapterSize, SizeType32 maxAdapterSize = kDefaultMaxAdapterSize, SizeType32 numPutWorkers = 1, SizeType32 numEnsureWorkers = 1, SizeType32 numCopyStreams = 1, SizeType32 maxPagesPerBlockHost = kDefaultMaxPagesPerBlockHost, SizeType32 maxPagesPerBlockDevice = kDefaultMaxPagesPerBlockDevice, std::optional<float> const &deviceCachePercent = std::nullopt, std::optional<size_t> const &hostCacheSize = std::nullopt)
-
+void setBadWords(std::list<VecTokens> const &badWords)
-
-
-
-bool operator==(PeftCacheConfig const &other) const
-
+void setStopWords(std::list<VecTokens> const &stopWords)
-
-
-
-SizeType32 getNumHostModuleLayer() const
-
+void setEmbeddingBias(Tensor const &embeddingBias)
-
-
-
-SizeType32 getNumDeviceModuleLayer() const
-
+void setExternalDraftTokensConfig(ExternalDraftTokensConfig const &externalDraftTokensConfig)
-
-
-
-SizeType32 getOptimalAdapterSize() const
-
+void setPromptTuningConfig(PromptTuningConfig const &pTuningConfig)
-
-
-
-SizeType32 getMaxAdapterSize() const
-
+void setMropeConfig(MropeConfig const &mRopeConfig)
-
-
-
-SizeType32 getNumPutWorkers() const
-
+void setLoraConfig(LoraConfig const &loraConfig)
-
-
-
-SizeType32 getNumEnsureWorkers() const
-
+void setLookaheadConfig(LookaheadDecodingConfig const &lookaheadConfig)
-
-
-
-SizeType32 getNumCopyStreams() const
-
+void setKvCacheRetentionConfig(KvCacheRetentionConfig const &kvCacheRetentionConfig)
-
-
-
-SizeType32 getMaxPagesPerBlockHost() const
-
+void setLogitsPostProcessorName(std::string const &logitsPostProcessorName)
-
-
-
-SizeType32 getMaxPagesPerBlockDevice() const
-
+void setEncoderInputTokenIds(VecTokens const &encoderInputTokenIds)
-
-
-
-std::optional<size_t> getHostCacheSize() const
-
+void setPriority(PriorityType priority)
--Public Static Attributes
--
-
-
-static constexpr SizeType32 kDefaultOptimalAdapterSize = 8
-
+void setReturnAllGeneratedTokens(bool returnAllGeneratedTokens)
-
-static constexpr SizeType32 kDefaultMaxAdapterSize = 64
-
+void setRequestType(RequestType const &requestType)
-
-static constexpr SizeType32 kDefaultMaxPagesPerBlockHost = 24
-
+void setContextPhaseParams(ContextPhaseParams contextPhaseParams)
-
-static constexpr SizeType32 kDefaultMaxPagesPerBlockDevice = 8
-
+
-
-
-
+
-
-
-
+
-
-
-+Private Members
--
-
-
-SizeType32 mNumHostModuleLayer
-
+void setEncoderOutputLength(SizeType32 encoderOutputLength)
-
-SizeType32 mNumDeviceModuleLayer
-
-SizeType32 mOptimalAdapterSize
-
+void setNumReturnSequences(SizeType32 numReturnSequences)
-
-SizeType32 mMaxAdapterSize
-
+void setEagleConfig(std::optional<EagleConfig> const &eagleConfig)
-
-SizeType32 mNumPutWorkers
-
-SizeType32 mNumEnsureWorkers
-
+void setGuidedDecodingParams(GuidedDecodingParams const &guidedDecodingParams)
-
-SizeType32 mNumCopyStreams
-
+void setAllottedTimeMs(MillisecondsType allottedTimeMs)
-
-SizeType32 mMaxPagesPerBlockHost
-
+void setAdditionalOutputNames(std::optional<std::vector<std::string>> additionalOutputNames)
-
+
-
-
-
-
-
+
-
-
-
+
-
-
-
-
-
+
-
-
-
+
-
-
-
+
++ @@ -6203,88 +6164,84 @@Public Static Attributes
-
-
-
-SizeType32 mMaxPagesPerBlockDevice
-
+static constexpr PriorityType kDefaultPriority = 0.5
- +static auto constexpr kBatchedPostProcessorName = "batched"
+This logits postprocessor name will dispatch to the batched logits postprocessor.
+executor.h -
- -class DecodingConfig
+- +class Response
#include <executor.h>-Configuration class for the decoding.
+Class that holds either an error or a result.
Public Functions
-
-
-
-explicit DecodingConfig(std::optional<DecodingMode> decodingMode = std::nullopt, std::optional<LookaheadDecodingConfig> lookaheadDecodingConfig = std::nullopt, std::optional<MedusaChoices> medusaChoices = std::nullopt, std::optional<EagleConfig> eagleConfig = std::nullopt)
-
+Response(IdType requestId, std::string errorMsg, std::optional<IdType> clientId = std::nullopt)
-
-
-
-bool operator==(DecodingConfig const &other) const
-
+Response(IdType requestId, Result Result, std::optional<IdType> clientId = std::nullopt)
-
-
-
-void setDecodingMode(DecodingMode const&)
Sets decoding mode. Some modes require the use of their own setters.
-
- +~Response()
+-
-
-
-std::optional<DecodingMode> getDecodingMode() const
-
+Response(Response const &other)
-
-
-
-void setLookaheadDecoding(LookaheadDecodingConfig const &lookaheadDecodingConfig)
Sets lookahead decoding mode and config.
-
- +Response(Response &&other) noexcept
+-
-
-
-std::optional<LookaheadDecodingConfig> getLookaheadDecodingConfig() const
-
+Response &operator=(Response const &other)
-
-
-
-void setMedusaChoices(MedusaChoices const&)
Sets medusa mode and config.
+-
+Response &operator=(Response &&other) noexcept
-
-
-
-std::optional<MedusaChoices> getMedusaChoices() const
- +std::optional<IdType> getClientId() const
+Get the client id of the request for which this response was generated.
+-
-
-
-void setEagleConfig(EagleConfig const&)
Sets eagle mode and config.
+-
+bool hasError() const
Indicates if this response has an error or not.
+
-
+
-
+std::string const &getErrorMsg() const
Get the error msg for this response Will throw an exception if hasError is false.
+
-
+
-
+Result const &getResult() const
Get the result for this response Will throw an exception if hasResult is true.
-
-
-
-std::optional<EagleConfig> getEagleConfig() const
@@ -6298,1854 +6255,2083 @@Private Members
-
-
-
-std::optional<DecodingMode> mDecodingMode
-
-
-
-std::optional<LookaheadDecodingConfig> mLookaheadDecodingConfig
-
-
-
-std::optional<MedusaChoices> mMedusaChoices
-
-
-
-std::optional<EagleConfig> mEagleConfig
-
+std::unique_ptr<Impl> mImpl
executor.h -
- -class GuidedDecodingConfig
+-
+
-
+struct Result
- #include <executor.h>-
Guided decoding configurations for executor.
+Struct that holds the generation result.
--Public Types
--
-
-
-enum class GuidedDecodingBackend
Values:
--
-
-
-enumerator kXGRAMMAR
Enable guided decoding with XGrammar backend.
+Public Members
+-
+
-
+bool isFinal
Indicates if this is the final result for the request.
-
+
-
+BeamTokens outputTokenIds
The output tokens for each beam.
-
+bool isFinal
-
-enumerator kXGRAMMAR
--Public Functions
--
-
-
-explicit GuidedDecodingConfig(GuidedDecodingBackend backend, std::optional<std::vector<std::string>> encodedVocab = std::nullopt, std::optional<std::string> tokenizerStr = std::nullopt, std::optional<std::vector<TokenIdType>> stopTokenIds = std::nullopt)
-
-
-
-bool operator==(GuidedDecodingConfig const &other) const
-
-
-
-void setBackend(GuidedDecodingBackend const &backend)
-
+
-
+std::optional<VecLogProbs> cumLogProbs
The cumulative log probabilities. Size beamSize.
+
-
-
-
-GuidedDecodingBackend getBackend() const
-
+
-
+std::optional<std::vector<VecLogProbs>> logProbs
The log probabilities for each generated token. Size [beamSize, outputLen].
+
-
-
-
-void setEncodedVocab(std::vector<std::string> const &encodedVocab)
-
-
-
-std::optional<std::vector<std::string>> getEncodedVocab() const
-
+
-
+std::optional<Tensor> generationLogits
The generation logits. Size [beamSize, maxNewTokens, vocabSizePadded] (non-streaming) or [maxNewTokens, beamSize, vocabSizePadded] (streaming and allGeneratedTokens) or [1, beamSize, vocabSizePadded] (streaming and non-allGeneratedTokens)
+
-
-
-
-void setTokenizerStr(std::string const &tokenizerStr)
-
+
-
+std::optional<SpeculativeDecodingFastLogitsInfo> specDecFastLogitsInfo
Logits information for direct transfer when using fast logits.
+
-
-
-
-std::optional<std::string> getTokenizerStr() const
-
-
-
-void setStopTokenIds(std::vector<TokenIdType> const &stopTokenIds)
-
+
-
+std::vector<FinishReason> finishReasons
The reason why the model stopped generating tokens for each beam in this request. Size [beamSize]. Currently only supported when beamSize is 1 and when using BatchingType::kINFLIGHT.
+
-
-
-
-std::optional<std::vector<TokenIdType>> getStopTokenIds() const
-
+
-
+std::optional<ContextPhaseParams> contextPhaseParams
The params of the context phase.
+
-
-
-
-void validate() const
-
+
-
+SizeType32 decodingIter = {0}
The number of the decoding iterations used to generate the result. In autoregressive decoding, it is equal to the maximum length of the beam in outputTokenIds. In speculative decoding, might be less than maximum length of the beam in outputTokenIds as more than one token can be generated per iteration. Used for speculative decoding statistics.
+
-Private Members
-
-
-
-GuidedDecodingBackend mBackend
Guided decoding backend. Currently supports XGrammar.
+-
+SizeType32 sequenceIndex = {0}
The index of the output sequence of this result where 0 <= sequenceIndex < numReturnSequences. In beam search (beamWidth > 1), this index will be always zero because all beams to be returned are included in this result.
-
-
-
-std::optional<std::vector<std::string>> mEncodedVocab
Encoded vocabulary. For a huggingface tokenizer, it can be extracted by:
-+-encoded_vocab = tokenizer.get_vocab() -encoded_vocab = [token for token, _ in sorted(encoded_vocab.items(), key=lambda x: x[1])] -
-
+bool isSequenceFinal
Indicates if this is the final result for a given sequence in the request In beam search (beamWidth > 1), the value will always equal to the value of isFinal.
-
-
-
-std::optional<std::string> mTokenizerStr
Tokenizer string. For a huggingface fast tokenizer, it can be extracted by:
-+-tokenizer_str = tokenizer.backend_tokenizer.to_str() -
-
+std::optional<RequestPerfMetrics> requestPerfMetrics
Performance metrics if returnPerfMetrics is set in OutputConfig.
-
-
-
-std::optional<std::vector<TokenIdType>> mStopTokenIds
Stop token ids. If not provided, it can be automatically detected.
+-
+std::vector<AdditionalOutput> additionalOutputs
The additional outputs.
+
-
-enum class GuidedDecodingBackend
-
+
-
+struct RetentionPriorityAndDuration
- +
Public Functions
+-
+
-
+inline RetentionPriorityAndDuration(std::optional<RetentionPriority> const &retentionPriority, std::optional<std::chrono::milliseconds> const &durationMs)
-Friends
--
-
- -friend class Serialization -
Public Members
+-
+
-
+std::optional<RetentionPriority> retentionPriority
-
+
-
+std::optional<std::chrono::milliseconds> durationMs
-
+inline RetentionPriorityAndDuration(std::optional<RetentionPriority> const &retentionPriority, std::optional<std::chrono::milliseconds> const &durationMs)
-
-
-
-class LogitsPostProcessorConfig
- +
- +class SamplingConfig
+Sampling configuration.
+-Public Functions
-
-
-
-explicit LogitsPostProcessorConfig(std::optional<LogitsPostProcessorMap> processorMap = std::nullopt, std::optional<LogitsPostProcessorBatched> processorBatched = std::nullopt, bool replicate = true)
- +explicit SamplingConfig(SizeType32 beamWidth = 1, std::optional<SizeType32> const &topK = std::nullopt, std::optional<FloatType> const &topP = std::nullopt, std::optional<FloatType> const &topPMin = std::nullopt, std::optional<TokenIdType> const &topPResetIds = std::nullopt, std::optional<FloatType> const &topPDecay = std::nullopt, std::optional<RandomSeedType> const &seed = std::nullopt, std::optional<FloatType> const &temperature = std::nullopt, std::optional<SizeType32> const &minTokens = std::nullopt, std::optional<FloatType> const &beamSearchDiversityRate = std::nullopt, std::optional<FloatType> const &repetitionPenalty = std::nullopt, std::optional<FloatType> const &presencePenalty = std::nullopt, std::optional<FloatType> const &frequencyPenalty = std::nullopt, std::optional<FloatType> const &lengthPenalty = std::nullopt, std::optional<SizeType32> const &earlyStopping = std::nullopt, std::optional<SizeType32> const &noRepeatNgramSize = std::nullopt, std::optional<SizeType32> const &numReturnSequences = std::nullopt)
+Constructor for SamplingConfig See description of parameters below.
+-
-
-
-std::optional<LogitsPostProcessorMap> getProcessorMap() const
-
+bool operator==(SamplingConfig const &other) const
-
-
-
-std::optional<LogitsPostProcessorBatched> getProcessorBatched() const
-
+SizeType32 getBeamWidth() const
-
-
-
-bool getReplicate() const
-
+SizeType32 getNumReturnBeams() const
-
-
-
-void setProcessorMap(LogitsPostProcessorMap const &processorMap)
-
+std::optional<SizeType32> getTopK() const
-
-
-
-void setProcessorBatched(LogitsPostProcessorBatched const &processorBatched)
-
+std::optional<FloatType> getTopP() const
--Private Members
--
-
-
-std::optional<LogitsPostProcessorMap> mProcessorMap
mapping from post processor names to non-batched post processors
-
-
-
-
-std::optional<LogitsPostProcessorBatched> mProcessorBatched
single batched post processor
-
-
-
-
-bool mReplicate
If set to true, logits post processor will run on all TP ranks in last PP rank.
-
-
+
-
+std::optional<SizeType32> getTopPResetIds() const
-
-
-
-class ExecutorConfig
- -#include <executor.h>-
Configuration class for the model executor.
--+Public Functions
-
-
-
-explicit ExecutorConfig(SizeType32 maxBeamWidth = 1, SchedulerConfig schedulerConfig = SchedulerConfig(), KvCacheConfig kvCacheConfig = KvCacheConfig(), bool enableChunkedContext = true, bool normalizeLogProbs = true, SizeType32 iterStatsMaxIterations = kDefaultIterStatsMaxIterations, SizeType32 requestStatsMaxIterations = kDefaultRequestStatsMaxIterations, BatchingType batchingType = BatchingType::kINFLIGHT, std::optional<SizeType32> maxBatchSize = std::nullopt, std::optional<SizeType32> maxNumTokens = std::nullopt, std::optional<ParallelConfig> parallelConfig = std::nullopt, std::optional<PeftCacheConfig> const &peftCacheConfig = std::nullopt, std::optional<LogitsPostProcessorConfig> logitsPostProcessorConfig = std::nullopt, std::optional<DecodingConfig> decodingConfig = std::nullopt, float gpuWeightsPercent = 1, std::optional<SizeType32> maxQueueSize = std::nullopt, ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig = ExtendedRuntimePerfKnobConfig(), std::optional<DebugConfig> debugConfig = std::nullopt, SizeType32 recvPollPeriodMs = 0, uint64_t maxSeqIdleMicroseconds = kDefaultMaxSeqIdleMicroseconds, std::optional<SpeculativeDecodingConfig> specDecConfig = std::nullopt, std::optional<GuidedDecodingConfig> guidedDecodingConfig = std::nullopt)
-
+std::optional<RandomSeedType> getSeed() const
-
-
-
-SizeType32 getMaxBeamWidth() const
-
+std::optional<RandomSeedType> getRandomSeed() const
-
-
-
-SchedulerConfig getSchedulerConfig() const
-
+std::optional<FloatType> getTemperature() const
-
-
-
-KvCacheConfig getKvCacheConfig() const
-
+std::optional<SizeType32> getMinTokens() const
-
-
-
-SchedulerConfig &getSchedulerConfigRef()
-
+std::optional<SizeType32> getMinLength() const
-
-
-
-KvCacheConfig &getKvCacheConfigRef()
-
+std::optional<FloatType> getBeamSearchDiversityRate() const
-
-
-
-SizeType32 getIterStatsMaxIterations() const
-
+std::optional<FloatType> getFrequencyPenalty() const
-
-
-
-SizeType32 getRequestStatsMaxIterations() const
-
+std::optional<FloatType> getLengthPenalty() const
-
-
-
-BatchingType getBatchingType() const
-
+std::optional<SizeType32> getEarlyStopping() const
-
-
-
-std::optional<SizeType32> getMaxBatchSize() const
-
+std::optional<SizeType32> getNoRepeatNgramSize() const
-
-
-
-std::optional<SizeType32> getMaxNumTokens() const
-
+std::optional<SizeType32> getNumReturnSequences() const
-
-
-
-std::optional<ParallelConfig> getParallelConfig() const
-
+void setBeamWidth(SizeType32 beamWidth)
-
-
-
-std::optional<PeftCacheConfig> getPeftCacheConfig() const
-
+void setTopK(std::optional<SizeType32> const &topK)
-
-
-
-std::optional<LogitsPostProcessorConfig> getLogitsPostProcessorConfig() const
-
+void setTopP(std::optional<FloatType> const &topP)
-
-
-
-std::optional<DecodingConfig> getDecodingConfig() const
-
+void setTopPMin(std::optional<FloatType> const &topPMin)
-
-
-
-float getGpuWeightsPercent() const
-
+void setTopPResetIds(std::optional<TokenIdType> const &topPResetIds)
-
-
-
-std::optional<SizeType32> getMaxQueueSize() const
-
+void setTopPDecay(std::optional<FloatType> const &topPDecay)
-
-
-
-ExtendedRuntimePerfKnobConfig getExtendedRuntimePerfKnobConfig() const
-
+void setSeed(std::optional<RandomSeedType> const &seed)
-
-
-
-std::optional<DebugConfig> getDebugConfig() const
-
+void setRandomSeed(std::optional<RandomSeedType> const &randomSeed)
-
-
-
-SizeType32 getRecvPollPeriodMs() const
-
+void setTemperature(std::optional<FloatType> const &temperature)
-
-
-
-uint64_t getMaxSeqIdleMicroseconds() const
-
+void setMinTokens(std::optional<SizeType32> const &minTokens)
-
-
-
-std::optional<SpeculativeDecodingConfig> getSpecDecConfig() const
-
+void setMinLength(std::optional<SizeType32> const &minLength)
-
-
-
-std::optional<GuidedDecodingConfig> getGuidedDecodingConfig() const
-
+void setBeamSearchDiversityRate(std::optional<FloatType> const &beamSearchDiversityRate)
-
-
-
-void setMaxBeamWidth(SizeType32 maxBeamWidth)
-
+void setRepetitionPenalty(std::optional<FloatType> const &repetitionPenalty)
-
-
-
-void setMaxBatchSize(SizeType32 maxBatchSize)
-
+void setPresencePenalty(std::optional<FloatType> const &presencePenalty)
-
-
-
-void setMaxNumTokens(SizeType32 maxNumTokens)
-
+void setFrequencyPenalty(std::optional<FloatType> const &frequencyPenalty)
-
-
-
-void setSchedulerConfig(SchedulerConfig const &schedulerConfig)
-
+void setLengthPenalty(std::optional<FloatType> const &lengthPenalty)
-
-
-
-void setKvCacheConfig(KvCacheConfig const &kvCacheConfig)
-
+void setEarlyStopping(std::optional<SizeType32> const &earlyStopping)
-
-
-
-void setEnableChunkedContext(bool enableChunkedContext)
-
+void setNoRepeatNgramSize(std::optional<SizeType32> const &noRepeatNgramSize)
-
-
-
-void setNormalizeLogProbs(bool normalizeLogProbs)
-
+void setNumReturnSequences(std::optional<SizeType32> const &numReturnSequences)
++Private Functions
-
-
-
-void setIterStatsMaxIterations(SizeType32 iterStatsMaxIterations)
-
+void updateNumReturnBeams()
++Private Members
+-
+
-
+SizeType32 mBeamWidth
The beam width. Default is 1 which disables beam search.
+
-
+
-
+std::optional<SizeType32> mTopK
Controls number of logits to sample from. Default is 0 (all logits).
+
-
+
-
+std::optional<FloatType> mTopP
Controls the top-P probability to sample from. Default is 0.f.
+
-
+
-
+std::optional<FloatType> mTopPMin
Controls decay in the top-P algorithm. topPMin is lower-bound. Default is 1.e-6.
+
-
+
-
+std::optional<TokenIdType> mTopPResetIds
Controls decay in the top-P algorithm. Indicates where to reset the decay. Default is 1.
+
-
+
-
+std::optional<FloatType> mTopPDecay
Controls decay in the top-P algorithm. The decay value. Default is 1.f.
+
-
+
-
+std::optional<RandomSeedType> mSeed
Controls the random seed used by the random number generator in sampling.
+
-
+
-
+std::optional<FloatType> mTemperature
Controls the modulation of logits when sampling new tokens. It can have values > 0.f. Default is 1.0f.
+
-
+
-
+std::optional<SizeType32> mMinTokens
Lower bound on the number of tokens to generate. Values < 1 have no effect. Default is 1.
+
-
+
-
+std::optional<FloatType> mRepetitionPenalty
Used to penalize tokens based on how often they appear in the sequence. It can have any value > 0.f. Values < 1.f encourages repetition, values > 1.f discourages it. Default is 1.f.
+
-
+
-
+std::optional<FloatType> mPresencePenalty
Used to penalize tokens already present in the sequence (irrespective of the number of appearances). It can have any values. Values < 0.f encourage repetition, values > 0.f discourage it. Default is 0.f.
+
-
+
-
+std::optional<FloatType> mFrequencyPenalty
Used to penalize tokens already present in the sequence (dependent on the number of appearances). It can have any values. Values < 0.f encourage repetition, values > 0.f discourage it. Default is 0.f.
+
-
+
-
+std::optional<FloatType> mLengthPenalty
Controls how to penalize longer sequences in beam search. Default is 0.f.
+
-
+
-
+std::optional<SizeType32> mEarlyStopping
Controls whether the generation process finishes once beamWidth sentences are generated (ends with end_token)
+
-
+
-
+std::optional<SizeType32> mNoRepeatNgramSize
Controls how many repeat ngram size are acceptable. Default is 1 << 30.
+
-
+
-
+std::optional<SizeType32> mNumReturnSequences
The number of return sequences or beams. In beam search, the value should be less than or equal to mBeamWidth. In sampling, it specifies the total number of independently generated sequences.
+
-
+
-
+SizeType32 mNumReturnBeams
The number of beams to return. It is equal to beamWidth unless numReturnSequences is set. If beamWidth > 1 and numReturnSequences is set, then numReturnBeams is equal to numReturnSequences.
+
++Private Static Functions
-
-
-
-void setRequestStatsMaxIterations(SizeType32 requestStatsMaxIterations)
-
+static SizeType32 checkBeamWidth(SizeType32 beamWidth)
-
-
-
-void setBatchingType(BatchingType batchingType)
-
+static std::optional<FloatType> const &checkTopK(std::optional<FloatType> const &topK)
-
-
-
-void setParallelConfig(ParallelConfig const ¶llelConfig)
-
+static std::optional<FloatType> const &checkTopP(std::optional<FloatType> const &topP)
-
-
-
-void setPeftCacheConfig(PeftCacheConfig const &peftCacheConfig)
-
+static std::optional<FloatType> const &checkTopPMin(std::optional<FloatType> const &topPMin)
-
-
-
-void setLogitsPostProcessorConfig(LogitsPostProcessorConfig const &logitsPostProcessorConfig)
-
+static std::optional<TokenIdType> const &checkTopPResetIds(std::optional<TokenIdType> const &topPResetIds)
-
-
-
-void setDecodingConfig(DecodingConfig const &decodingConfig)
-
+static std::optional<FloatType> const &checkTopPDecay(std::optional<FloatType> const &topPDecay)
-
-
-
-void setGpuWeightsPercent(float const &gpuWeightsPercent)
-
+static std::optional<FloatType> const &checkTemperature(std::optional<FloatType> const &temperature)
-
-
-
-void setMaxQueueSize(std::optional<SizeType32> const &maxQueueSize)
-
+static std::optional<FloatType> const &checkRepetitionPenalty(std::optional<FloatType> const &penalty)
-
-
-
-void setExtendedRuntimePerfKnobConfig(ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig)
-
+static std::optional<SizeType32> const &checkMinTokens(std::optional<SizeType32> const &minTokens)
-
-
-
-void setDebugConfig(DebugConfig const &debugConfig)
-
+static std::optional<SizeType32> const &checkNoRepeatNgramSize(std::optional<SizeType32> const &noRepeatNgramSize)
-
-
-
-void setRecvPollPeriodMs(SizeType32 const &recvPollPeriodMs)
-
+static std::optional<FloatType> const &checkBeamSearchDiversityRate(std::optional<FloatType> const &beamSearchDiversityRate)
-
-
-
-void setMaxSeqIdleMicroseconds(uint64_t maxNumTokens)
-
+static std::optional<SizeType32> const &checkNumReturnSequences(std::optional<SizeType32> const &numReturnSequences, SizeType32 beamWidth)
++Friends
+-
+
- +friend class Serialization +
-
-explicit ExecutorConfig(SizeType32 maxBeamWidth = 1, SchedulerConfig schedulerConfig = SchedulerConfig(), KvCacheConfig kvCacheConfig = KvCacheConfig(), bool enableChunkedContext = true, bool normalizeLogProbs = true, SizeType32 iterStatsMaxIterations = kDefaultIterStatsMaxIterations, SizeType32 requestStatsMaxIterations = kDefaultRequestStatsMaxIterations, BatchingType batchingType = BatchingType::kINFLIGHT, std::optional<SizeType32> maxBatchSize = std::nullopt, std::optional<SizeType32> maxNumTokens = std::nullopt, std::optional<ParallelConfig> parallelConfig = std::nullopt, std::optional<PeftCacheConfig> const &peftCacheConfig = std::nullopt, std::optional<LogitsPostProcessorConfig> logitsPostProcessorConfig = std::nullopt, std::optional<DecodingConfig> decodingConfig = std::nullopt, float gpuWeightsPercent = 1, std::optional<SizeType32> maxQueueSize = std::nullopt, ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig = ExtendedRuntimePerfKnobConfig(), std::optional<DebugConfig> debugConfig = std::nullopt, SizeType32 recvPollPeriodMs = 0, uint64_t maxSeqIdleMicroseconds = kDefaultMaxSeqIdleMicroseconds, std::optional<SpeculativeDecodingConfig> specDecConfig = std::nullopt, std::optional<GuidedDecodingConfig> guidedDecodingConfig = std::nullopt)
-
+
-
+class SchedulerConfig
- +#include <executor.h>+
Configuration class for the scheduler.
++-Public Functions
-
-
-
-void setSpecDecConfig(SpeculativeDecodingConfig const &specDecConfig)
-
+explicit SchedulerConfig(CapacitySchedulerPolicy capacitySchedulerPolicy = CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT, std::optional<ContextChunkingPolicy> contextChunkingPolicy = std::nullopt, std::optional<DynamicBatchConfig> dynamicBatchConfig = std::nullopt)
-
-
-
-void setGuidedDecodingConfig(GuidedDecodingConfig const &guidedDecodingConfig)
-
+bool operator==(SchedulerConfig const &other) const
-Public Static Attributes
--
-
-
-static constexpr uint64_t kDefaultMaxSeqIdleMicroseconds = 180000000
-
+CapacitySchedulerPolicy getCapacitySchedulerPolicy() const
-
-static constexpr SizeType32 kDefaultIterStatsMaxIterations = 1000
-
+std::optional<ContextChunkingPolicy> getContextChunkingPolicy() const
-
-static constexpr SizeType32 kDefaultRequestStatsMaxIterations = 0
-
+std::optional<DynamicBatchConfig> getDynamicBatchConfig() const
-
+
-
-
-
+
-
-
-
+
+Private Members
-
-
-
-SizeType32 mMaxBeamWidth
The beam width value of requests that will be sent to the executor.
+-
+CapacitySchedulerPolicy mCapacitySchedulerPolicy
The capacity scheduler policy. See CapacitySchedulerPolicy.
-
-
-
-SchedulerConfig mSchedulerConfig
The scheduler configuration.
+-
+std::optional<ContextChunkingPolicy> mContextChunkingPolicy
The context chunking policy. See ContextChunkingPolicy.
-
-
-
-KvCacheConfig mKvCacheConfig
The KV cache configuration.
+-
+std::optional<DynamicBatchConfig> mDynamicBatchConfig
The config for tuning batch size dynamically. See DynamicBatchSizeConfig.
-
-
-
-bool mEnableChunkedContext
The KV cache configuration.
+
+Friends
+-
+
- +friend class Serialization +
-
-void setSpecDecConfig(SpeculativeDecodingConfig const &specDecConfig)
-
+
-
+class SpeculativeDecodingConfig
- +#include <executor.h>+
Configuration for speculative decoding (both draft and target models)
+++Public Functions
+-
+
-
+explicit SpeculativeDecodingConfig(bool fastLogits = false)
-
+
-
+bool operator==(SpeculativeDecodingConfig const &other) const
+Public Members
-
-
-
-bool mNormalizeLogProbs
Controls if log probabilities should be normalized or not.
+-
+bool fastLogits
Send logits tensor directly from draft to target model.
-
-
-
-SizeType32 mIterStatsMaxIterations
Controls the maximum number of iterations for which to keep statistics.
+
-
+explicit SpeculativeDecodingConfig(bool fastLogits = false)
-
-
-
-SizeType32 mRequestStatsMaxIterations
Controls the maximum number of iterations for which to keep per-request statistics.
+-
+
-
+struct SpeculativeDecodingFastLogitsInfo
- +#include <executor.h>+
Struct that holds the logits information when using direct transfer.
+++Public Functions
+ ++Public Members
-
-
-
-BatchingType mBatchingType
The type of batching strategy to use. See BatchingType.
+-
+uint64_t draftRequestId
Draft request id.
-
-
-
-std::optional<SizeType32> mMaxBatchSize
The max batch size of requests.
+-
+int32_t draftParticipantId
MPI world rank of the draft model leader.
-
-
-
-std::optional<SizeType32> mMaxNumTokens
The max number of tokens per batch.
+
-
-BatchingType mBatchingType
-
-
-
-std::optional<ParallelConfig> mParallelConfig
The parallel execution configuration.
-
-
-
-std::optional<PeftCacheConfig> mPeftCacheConfig
-
+namespace mpi
-
-std::optional<LogitsPostProcessorConfig> mLogitsPostProcessorConfig
Logits post processor configuration.
-
-std::optional<DecodingConfig> mDecodingConfig
Decoding configuration.
--
+namespace tensorrt_llm
-
+
-
+namespace executor
-
+
-
+class Serialization
- +-
Public Static Functions
+-
+
-
+static RequestPerfMetrics::TimePoint deserializeTimePoint(std::istream &is)
-
-
-
-float mGpuWeightsPercent
GPU weights percent for weight streaming.
-
-
+
-
+static void serialize(RequestPerfMetrics::TimePoint const &tp, std::ostream &os)
-
-
-
-std::optional<SizeType32> mMaxQueueSize
The maximum number of requests allowed in queue before rejecting new requests.
-
-
+
-
+static size_t serializedSize(RequestPerfMetrics::TimePoint const&)
-
-
-
-ExtendedRuntimePerfKnobConfig mExtendedRuntimePerfKnobConfig
Config for perf knobs that can be set in runtime.
-
-
+
-
+static RequestPerfMetrics deserializeRequestPerfMetrics(std::istream &is)
-
-
-
-std::optional<DebugConfig> mDebugConfig
Debugging configuration.
-
-
+
-
+static void serialize(RequestPerfMetrics const &metrics, std::ostream &os)
-
+
-
+static size_t serializedSize(RequestPerfMetrics const &metrics)
-
+
-
+static SamplingConfig deserializeSamplingConfig(std::istream &is)
-
+
-
+static void serialize(SamplingConfig const &config, std::ostream &os)
-
+
-
+static size_t serializedSize(SamplingConfig const &config)
-
+
-
+static OutputConfig deserializeOutputConfig(std::istream &is)
-
-
-
-SizeType32 mRecvPollPeriodMs
The time in ms between polls for new communication in orchestrator mode. Use 0 for busy loop.
-
-
+
-
+static void serialize(OutputConfig const &config, std::ostream &os)
-
-
-
-uint64_t mMaxSeqIdleMicroseconds
The maximum time in microseconds a scheduled request can remain idle before getting terminated. Default is 3 minutes.
-
-
+
-
+static size_t serializedSize(OutputConfig const &config)
-
-
-
-std::optional<SpeculativeDecodingConfig> mSpeculativeDecodingConfig
The speculative decoding configuration.
-
-
+
-
+static OutputConfig::AdditionalModelOutput deserializeAdditionalModelOutput(std::istream &is)
-
-
-
-std::optional<GuidedDecodingConfig> mGuidedDecodingConfig
The guided decoding configuration.
-
-
+
-
+static void serialize(OutputConfig::AdditionalModelOutput const &additionalModelOutput, std::ostream &os)
--Friends
--
-
- -friend class Serialization -
-
+
-
+static size_t serializedSize(OutputConfig::AdditionalModelOutput const &additionalModelOutput)
-
+static RequestPerfMetrics::TimePoint deserializeTimePoint(std::istream &is)
-
+
-
+static ExternalDraftTokensConfig deserializeExternalDraftTokensConfig(std::istream &is)
-
-
-
-struct KVCacheCreatedData
- --
Public Members
--
-
-
-std::vector<SizeType32> numBlocksPerCacheLevel
The amount of blocks at each cache level.
-
-
+
-
+static void serialize(ExternalDraftTokensConfig const &config, std::ostream &os)
-
-std::vector<SizeType32> numBlocksPerCacheLevel
-
+
-
+static size_t serializedSize(ExternalDraftTokensConfig const &config)
-
-
-
-struct KVCacheStoredBlockData
- -#include <executor.h>-
An entry for a single block stored into the tree.
---Public Functions
-
-
-
-inline KVCacheStoredBlockData(IdType blockHash, tensorrt_llm::runtime::VecUniqueTokens tokens, tensorrt_llm::runtime::LoraTaskIdType loraId, SizeType32 cacheLevel, SizeType32 priority)
-
+static PromptTuningConfig deserializePromptTuningConfig(std::istream &is)
--Public Members
- +-
+
-
+static void serialize(PromptTuningConfig const &config, std::ostream &os)
-
-
-
-tensorrt_llm::runtime::VecUniqueTokens tokens
The unique tokens of the block.
-
-
+
-
+static size_t serializedSize(PromptTuningConfig const &config)
-
-
-
-tensorrt_llm::runtime::LoraTaskIdType loraId
The Lora task id of the block.
-
-
+
-
+static MropeConfig deserializeMropeConfig(std::istream &is)
-
-
-
-SizeType32 cacheLevel
The cache level of the block.
-
-
+
-
+static void serialize(MropeConfig const &config, std::ostream &os)
-
-
-
-SizeType32 priority
The priority of the block.
-
-
+
-
+static size_t serializedSize(MropeConfig const &config)
-
-inline KVCacheStoredBlockData(IdType blockHash, tensorrt_llm::runtime::VecUniqueTokens tokens, tensorrt_llm::runtime::LoraTaskIdType loraId, SizeType32 cacheLevel, SizeType32 priority)
-
+
-
+static LoraConfig deserializeLoraConfig(std::istream &is)
-
-
-
-struct KVCacheStoredData
- --
Public Members
- +-
+
-
+static void serialize(LoraConfig const &config, std::ostream &os)
-
-
-
-std::vector<KVCacheStoredBlockData> blocks
A sequence of blocks. The parent of block
-iis blocki-1
-
+
-
+static size_t serializedSize(LoraConfig const &config)
-
+static void serialize(LoraConfig const &config, std::ostream &os)
-
-
-
-struct KVCacheRemovedData
- -
-
-
-
-template<typename T>
struct KVCacheEventDiff
- - -
-
-
-
-struct KVCacheUpdatedData
- --
Public Functions
-
-
-
-inline explicit KVCacheUpdatedData(IdType blockHash)
-
+static kv_cache::CacheState deserializeCacheState(std::istream &is)
-
-
-
-inline KVCacheUpdatedData &cacheLevelUpdated(SizeType32 oldValue, SizeType32 newValue)
-
+static void serialize(kv_cache::CacheState const &state, std::ostream &os)
-
-
-
-inline KVCacheUpdatedData &priorityUpdated(SizeType32 oldValue, SizeType32 newValue)
-
+static size_t serializedSize(kv_cache::CacheState const &state)
--Public Members
- +-
+
-
+static DataTransceiverState deserializeDataTransceiverState(std::istream &is)
-
-
-
-std::optional<KVCacheEventDiff<SizeType32>> cacheLevel = std::nullopt
The updated value of the cacheLevel field.
-
-
+
-
+static void serialize(DataTransceiverState const &dataTransceiverState, std::ostream &os)
-
-
-
-std::optional<KVCacheEventDiff<SizeType32>> priority = std::nullopt
The updated value of the priority field.
-
-
+
-
+static size_t serializedSize(DataTransceiverState const &dataTransceiverState)
-
+
-
+static ContextPhaseParams deserializeContextPhaseParams(std::istream &is)
-
-inline explicit KVCacheUpdatedData(IdType blockHash)
-
+
-
+static void serialize(ContextPhaseParams const &contextPhaseParams, std::ostream &os)
-
-
-
-struct KVCacheEvent
- -- -
Public Functions
-
-
-
-KVCacheEvent(IdType eventId, KVCacheEventData data)
-
+static size_t serializedSize(ContextPhaseParams const &contextPhaseParams)
-
-KVCacheEvent(IdType eventId, KVCacheEventData data)
-
-
-
-class KVCacheEventManager
- -#include <executor.h>-
Exposes a limited set of KV cache manager functionalities.
---Public Functions
-
-
-
-std::deque<KVCacheEvent> getLatestEvents(std::optional<std::chrono::milliseconds> timeout = std::nullopt)
Get the latest KV Cache events.
--
-
- Parameters: -
timeout – The maximum time to wait for new events. If nullopt, will only return when new events are available, or when the executor instance has shutdown.
-
-
- +static void serialize(Tensor const &tensor, std::ostream &os)
+ ---Private Members
--
-
-
-std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager> kvCacheManager
-
-std::deque<KVCacheEvent> getLatestEvents(std::optional<std::chrono::milliseconds> timeout = std::nullopt)
-
+
-
+static SpeculativeDecodingFastLogitsInfo deserializeSpecDecFastLogitsInfo(std::istream &is)
-
-
-
-class Executor
- -#include <executor.h>-
The executor is responsible for receiving new requests and sending responses, and running the inference.
---Public Functions
-
-
-
-Executor(std::filesystem::path const &modelPath, ModelType modelType, ExecutorConfig const &executorConfig)
-
-
- Parameters: -
-
-
modelPath – Path to the folder that defines the model to run
-modelType – The type of model
-executorConfig – The configuration for the executor
-comm – An optional inter-process communicator configuration
-
-
- +static void serialize(SpeculativeDecodingFastLogitsInfo const &info, std::ostream &os)
+-
-
-
-Executor(std::filesystem::path const &encoderModelPath, std::filesystem::path const &decoderModelPath, ModelType modelType, ExecutorConfig const &executorConfig)
-
+static size_t serializedSize(SpeculativeDecodingFastLogitsInfo const &info)
-
-
-
-Executor(BufferView const &engineBuffer, std::string const &jsonConfigStr, ModelType modelType, ExecutorConfig const &executorConfig, std::optional<std::map<std::string, Tensor>> const &managedWeights = std::nullopt)
-
+static Result deserializeResult(std::istream &is)
-
-
-
-Executor(BufferView const &encoderEngineBuffer, std::string const &encoderJsonConfigStr, BufferView const &decoderEngineBuffer, std::string const &decoderJsonConfigStr, ModelType modelType, ExecutorConfig const &executorConfig)
-
+static void serialize(Result const &result, std::ostream &os)
-
-
+
-
+static AdditionalOutput deserializeAdditionalOutput(std::istream &is)
-
-
-
-~Executor()
-
+static void serialize(AdditionalOutput const &additionalOutput, std::ostream &os)
-
-
-
-Executor(Executor const &executor) = delete
-
+static size_t serializedSize(AdditionalOutput const &additionalOutput)
-
-
-
-Executor &operator=(Executor const &executor) = delete
-
+static Response deserializeResponse(std::istream &is)
-
-
-
-Executor(Executor&&) = default
-
+static void serialize(Response const &response, std::ostream &os)
-
-
-
-Executor &operator=(Executor&&) = default
-
+static size_t serializedSize(Response const &response)
-
-
-
-IdType enqueueRequest(Request const &request)
Enqueue a new request.
--
-
- Parameters: -
request – The LLM request which contains input tokens and request parameters
-
-- Returns: -
A unique id that identifies the request
-
-
- +static std::vector<Response> deserializeResponses(std::vector<char> &buffer)
+-
-
-
-std::vector<IdType> enqueueRequests(std::vector<Request> const &requests)
Enqueue a batch of request.
-
- +static std::vector<char> serialize(std::vector<Response> const &responses)
+-
-
-
-std::vector<Response> awaitResponses(std::optional<std::chrono::milliseconds> const &timeout = std::nullopt)
Await for ready responses.
-- --This overload awaits for any ready responses. In particular, if several requests - have been enqueued, this method will provide any ready responses without order guarantees. -
-
-
- Parameters: -
timeout – The maximum time to wait for new responses
-
-- Returns: -
A vector of responses
-
-
- +static KvCacheConfig deserializeKvCacheConfig(std::istream &is)
+-
-
-
-std::vector<Response> awaitResponses(IdType const &requestId, std::optional<std::chrono::milliseconds> const &timeout = std::nullopt)
Await for ready responses.
--
-
- Parameters: -
-
-
id – A request id
-timeout – The maximum time to wait for new responses
-
-- Returns: -
A vector of responses
-
-
- +static void serialize(KvCacheConfig const &kvCacheConfig, std::ostream &os)
+-
-
-
-std::vector<std::vector<Response>> awaitResponses(std::vector<IdType> const &requestIds, std::optional<std::chrono::milliseconds> const &timeout = std::nullopt)
Await for multiple ready responses.
-- --A multiple ID request behaves as if awaitResponses(IdType, timeout) - were invoked on all IDs. The returned vector contains - a vector of responses per ID in the same order specified by the requestIds. - The same behaviour as awaitResponses(IdType, timeout) applies: - * Responses may be empty. - * If all responses have already been given for one of the requestIds, - then this method will hang unless a timeout is specified. -
-
-
- Parameters: -
-
-
requestIds – Ids requested
-timeout – The maximum time to wait for new responses
-
-- Returns: -
A vector of vector of responses
-
-
- +static size_t serializedSize(KvCacheConfig const &kvCacheConfig)
+-
-
-
-SizeType32 getNumResponsesReady(std::optional<IdType> const &requestId = std::nullopt) const
Get the number of ready responses.
--
-
- Parameters: -
requestId – An optional request id
-
-- Returns: -
The number of ready responses
-
-
- +static DynamicBatchConfig deserializeDynamicBatchConfig(std::istream &is)
+-
-
-
-void cancelRequest(IdType requestId)
Cancel the request with provided request id.
--
-
- Parameters: -
id – The request id for which to cancel the response
-
-
- +static void serialize(DynamicBatchConfig const &dynamicBatchConfig, std::ostream &os)
+-
-
-
-void shutdown()
Signals the server to shutdown.
-This call is blocking. Only returns when all requests have terminated or timeout has been reached
-
- +static size_t serializedSize(DynamicBatchConfig const &dynamicBatchConfig)
+ + +-
+
-
+static SchedulerConfig deserializeSchedulerConfig(std::istream &is)
-
-
-
-std::deque<IterationStats> getLatestIterationStats()
Returns the per-iterations statistics computed since last call to getLatestIterationStats. Contains at most iterStatsMaxIterations iterations.
--
-
- Returns: -
Iteration stats
-
-
- +static void serialize(SchedulerConfig const &schedulerConfig, std::ostream &os)
+-
-
-
-std::deque<RequestStatsPerIteration> getLatestRequestStats()
Returns the request stats of each iteration computed since last call to getLatestRequestStats. Contains at most requestStatsMaxIterations iterations.
--
-
- Returns: -
Request stats grouped by iterations
-
-
- +static size_t serializedSize(SchedulerConfig const &schedulerConfig)
+-
-
-
-std::deque<DebugTensorsPerIteration> getLatestDebugTensors()
Returns the debug tensors of each iteration computed since last call to getLatestDebugTensors. Contains at most debugTensorsMaxIterations iterations.
--
-
- Returns: -
Request debug tensors grouped by iterations
-
-
- +static ExtendedRuntimePerfKnobConfig deserializeExtendedRuntimePerfKnobConfig(std::istream &is)
+-
-
-
-bool canEnqueueRequests() const
Indicates if the current process is allowed to enqueueRequests.
-
- +static void serialize(ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig, std::ostream &os)
+-
-
-
-bool isParticipant() const
Indicates if the current process participates in this executor instance.
-
- +static size_t serializedSize(ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig)
+-
-
-
-std::optional<std::shared_ptr<KVCacheEventManager>> getKVCacheEventManager() const
-
+static ParallelConfig deserializeParallelConfig(std::istream &is)
--Private Members
--
-
-
-std::unique_ptr<Impl> mImpl
-
+static void serialize(ParallelConfig const ¶llelConfig, std::ostream &os)
-
+
-
-Executor(std::filesystem::path const &modelPath, ModelType modelType, ExecutorConfig const &executorConfig)
-
-
-
-class JsonSerialization
- -#include <executor.h>-
Class with utility functions to serialize statistics to json string.
---Public Static Functions
-
-
-
-static std::string toJsonStr(IterationStats const &iterationStats)
Utility function to convert an iterationStats struct to a json serialized string.
-
- +static size_t serializedSize(ParallelConfig const ¶llelConfig)
+-
-
-
-static std::string toJsonStr(RequestStatsPerIteration const &requestStatsPerIter)
Utility function to convert a requestStatsPerIteration struct to a json serialized string.
-
- +static PeftCacheConfig deserializePeftCacheConfig(std::istream &is)
+-
-
-
-static std::string toJsonStr(RequestStats const &requestStats)
Utility function to convert a requestStats struct to a json serialized string.
-
- +static void serialize(PeftCacheConfig const &peftCacheConfig, std::ostream &os)
+ - -
-static std::string toJsonStr(IterationStats const &iterationStats)
-
+
-
+static size_t serializedSize(PeftCacheConfig const &peftCacheConfig)
-
+class Serialization
-
+
-
+static OrchestratorConfig deserializeOrchestratorConfig(std::istream &is)
-
-
-
-namespace mpi
-
+static void serialize(OrchestratorConfig const &orchestratorConfig, std::ostream &os)
-
+
-
+namespace executor
-
+static size_t serializedSize(OrchestratorConfig const &orchestratorConfig)
-
-namespace tensorrt_llm
-
-
-
-namespace executor
-
-
-
-class Serialization
- -+
Public Static Functions
-
-
-
-static RequestPerfMetrics::TimePoint deserializeTimePoint(std::istream &is)
-
+static DecodingMode deserializeDecodingMode(std::istream &is)
-
-
-
-static void serialize(RequestPerfMetrics::TimePoint const &tp, std::ostream &os)
-
+static void serialize(DecodingMode const &decodingMode, std::ostream &os)
-
-
-
-static size_t serializedSize(RequestPerfMetrics::TimePoint const&)
-
+static size_t serializedSize(DecodingMode const &decodingMode)
-
-
-
-static RequestPerfMetrics deserializeRequestPerfMetrics(std::istream &is)
-
+static LookaheadDecodingConfig deserializeLookaheadDecodingConfig(std::istream &is)
-
-
-
-static void serialize(RequestPerfMetrics const &metrics, std::ostream &os)
-
+static void serialize(LookaheadDecodingConfig const &lookaheadDecodingConfig, std::ostream &os)
-
-
-
-static size_t serializedSize(RequestPerfMetrics const &metrics)
-
+static size_t serializedSize(LookaheadDecodingConfig const &lookaheadDecodingConfig)
-
-
-
-static SamplingConfig deserializeSamplingConfig(std::istream &is)
-
+static EagleConfig deserializeEagleConfig(std::istream &is)
-
-
-
-static void serialize(SamplingConfig const &config, std::ostream &os)
-
+static void serialize(EagleConfig const &eagleConfig, std::ostream &os)
-
-
-
-static size_t serializedSize(SamplingConfig const &config)
-
+static size_t serializedSize(EagleConfig const &eagleConfig)
-
-
-
-static OutputConfig deserializeOutputConfig(std::istream &is)
-
+static SpeculativeDecodingConfig deserializeSpeculativeDecodingConfig(std::istream &is)
-
-
-
-static void serialize(OutputConfig const &config, std::ostream &os)
-
+static void serialize(SpeculativeDecodingConfig const &specDecConfig, std::ostream &os)
-
-
-
-static size_t serializedSize(OutputConfig const &config)
-
+static size_t serializedSize(SpeculativeDecodingConfig const &specDecConfig)
-
-
-
-static ExternalDraftTokensConfig deserializeExternalDraftTokensConfig(std::istream &is)
-
+static GuidedDecodingConfig deserializeGuidedDecodingConfig(std::istream &is)
-
-
-
-static void serialize(ExternalDraftTokensConfig const &config, std::ostream &os)
-
+static void serialize(GuidedDecodingConfig const &guidedDecodingConfig, std::ostream &os)
-
-
-
-static size_t serializedSize(ExternalDraftTokensConfig const &config)
-
+static size_t serializedSize(GuidedDecodingConfig const &guidedDecodingConfig)
-
-
-
-static PromptTuningConfig deserializePromptTuningConfig(std::istream &is)
-
+static GuidedDecodingParams deserializeGuidedDecodingParams(std::istream &is)
-
-
-
-static void serialize(PromptTuningConfig const &config, std::ostream &os)
-
+static void serialize(GuidedDecodingParams const &guidedDecodingParams, std::ostream &os)
-
-
-
-static size_t serializedSize(PromptTuningConfig const &config)
-
+static size_t serializedSize(GuidedDecodingParams const &guidedDecodingParams)
-
-
-
-static MropeConfig deserializeMropeConfig(std::istream &is)
-
+static KvCacheRetentionConfig deserializeKvCacheRetentionConfig(std::istream &is)
-
-
-
-static void serialize(MropeConfig const &config, std::ostream &os)
-
+static void serialize(KvCacheRetentionConfig const &kvCacheRetentionConfig, std::ostream &os)
-
-
-
-static size_t serializedSize(MropeConfig const &config)
-
+static size_t serializedSize(KvCacheRetentionConfig const &kvCacheRetentionConfig)
-
-
-
-static LoraConfig deserializeLoraConfig(std::istream &is)
-
+static KvCacheRetentionConfig::TokenRangeRetentionConfig deserializeTokenRangeRetentionConfig(std::istream &is)
-
-
-
-static void serialize(LoraConfig const &config, std::ostream &os)
-
+static void serialize(KvCacheRetentionConfig::TokenRangeRetentionConfig const &tokenRangeRetentionConfig, std::ostream &os)
-
-
-
-static size_t serializedSize(LoraConfig const &config)
-
+static size_t serializedSize(KvCacheRetentionConfig::TokenRangeRetentionConfig const &tokenRangeRetentionConfig)
-
-
-
-static kv_cache::CommState deserializeCommState(std::istream &is)
-
+static DecodingConfig deserializeDecodingConfig(std::istream &is)
-
-
-
-static void serialize(kv_cache::CommState const &state, std::ostream &os)
-
+static void serialize(DecodingConfig const &decodingConfig, std::ostream &os)
-
-
-
-static size_t serializedSize(kv_cache::CommState const &state)
-
+static size_t serializedSize(DecodingConfig const &decodingConfig)
-
-
-
-static kv_cache::SocketState deserializeSocketState(std::istream &is)
-
+static DebugConfig deserializeDebugConfig(std::istream &is)
-
-
-
-static void serialize(kv_cache::SocketState const &state, std::ostream &os)
-
+static void serialize(DebugConfig const &debugConfig, std::ostream &os)
-
-
-
-static size_t serializedSize(kv_cache::SocketState const &state)
-
+static size_t serializedSize(DebugConfig const &debugConfig)
-
-
-
-static kv_cache::CacheState deserializeCacheState(std::istream &is)
-
+static ExecutorConfig deserializeExecutorConfig(std::istream &is)
-
-
-
-static void serialize(kv_cache::CacheState const &state, std::ostream &os)
-
+static void serialize(ExecutorConfig const &executorConfig, std::ostream &os)
-
-
-
-static size_t serializedSize(kv_cache::CacheState const &state)
-
+static size_t serializedSize(ExecutorConfig const &executorConfig)
-
-
-
-static DataTransceiverState deserializeDataTransceiverState(std::istream &is)
-
+static KvCacheStats deserializeKvCacheStats(std::istream &is)
-
-
-
-static void serialize(DataTransceiverState const &dataTransceiverState, std::ostream &os)
-
+static void serialize(KvCacheStats const &kvCacheStats, std::ostream &os)
-
-
-
-static size_t serializedSize(DataTransceiverState const &dataTransceiverState)
-
+static size_t serializedSize(KvCacheStats const &kvCacheStats)
-
-
-
-static ContextPhaseParams deserializeContextPhaseParams(std::istream &is)
-
+static StaticBatchingStats deserializeStaticBatchingStats(std::istream &is)
-
-
-
-static void serialize(ContextPhaseParams const &contextPhaseParams, std::ostream &os)
-
+static void serialize(StaticBatchingStats const &staticBatchingStats, std::ostream &os)
-
-
-
-static size_t serializedSize(ContextPhaseParams const &contextPhaseParams)
-
+static size_t serializedSize(StaticBatchingStats const &staticBatchingStats)
-
-
-
-static Request deserializeRequest(std::istream &is)
-
+static InflightBatchingStats deserializeInflightBatchingStats(std::istream &is)
-
-
-
-static void serialize(Request const &request, std::ostream &os)
-
+static void serialize(InflightBatchingStats const &inflightBatchingStats, std::ostream &os)
-
-
-
-static size_t serializedSize(Request const &request)
-
+static size_t serializedSize(InflightBatchingStats const &inflightBatchingStats)
-
-
-
-static Tensor deserializeTensor(std::istream &is)
-
+static IterationStats deserializeIterationStats(std::vector<char> &buffer)
-
-
-
-static void serialize(Tensor const &tensor, std::ostream &os)
-
+static IterationStats deserializeIterationStats(std::istream &is)
-
-
-
-static size_t serializedSize(Tensor const &tensor)
-
+static void serialize(IterationStats const &iterStats, std::ostream &os)
-
-
-
-static SpeculativeDecodingFastLogitsInfo deserializeSpecDecFastLogitsInfo(std::istream &is)
-
+static std::vector<char> serialize(IterationStats const &iterStats)
-
-
-
-static void serialize(SpeculativeDecodingFastLogitsInfo const &info, std::ostream &os)
-
+static size_t serializedSize(IterationStats const &iterStats)
-
-
-
-static size_t serializedSize(SpeculativeDecodingFastLogitsInfo const &info)
-
+static std::vector<char> serialize(std::vector<IterationStats> const &iterStatsVec)
-
-
-
-static Result deserializeResult(std::istream &is)
-
+static std::vector<IterationStats> deserializeIterationStatsVec(std::vector<char> &buffer)
-
-
-
-static void serialize(Result const &result, std::ostream &os)
-
+static DisServingRequestStats deserializeDisServingRequestStats(std::istream &is)
-
-
-
-static size_t serializedSize(Result const &result)
-
+static void serialize(DisServingRequestStats const &stats, std::ostream &os)
-
-
-
-static Response deserializeResponse(std::istream &is)
-
+static size_t serializedSize(DisServingRequestStats const &disServingRequestStats)
-
-
-
-static void serialize(Response const &response, std::ostream &os)
-
+static RequestStage deserializeRequestStage(std::istream &is)
-
-
-
-static size_t serializedSize(Response const &response)
-
+static void serialize(RequestStage const &requestStage, std::ostream &os)
-
-
-
-static std::vector<Response> deserializeResponses(std::vector<char> &buffer)
-
+static size_t serializedSize(RequestStage const &requestStage)
-
-
-
-static std::vector<char> serialize(std::vector<Response> const &responses)
-
+static RequestStats deserializeRequestStats(std::istream &is)
-
-
-
-static KvCacheConfig deserializeKvCacheConfig(std::istream &is)
-
+static void serialize(RequestStats const &state, std::ostream &os)
-
-
-
-static void serialize(KvCacheConfig const &kvCacheConfig, std::ostream &os)
-
+static size_t serializedSize(RequestStats const &state)
-
-
-
-static size_t serializedSize(KvCacheConfig const &kvCacheConfig)
-
+static RequestStatsPerIteration deserializeRequestStatsPerIteration(std::istream &is)
-
-
-
-static DynamicBatchConfig deserializeDynamicBatchConfig(std::istream &is)
-
+static RequestStatsPerIteration deserializeRequestStatsPerIteration(std::vector<char> &buffer)
-
-
-
-static void serialize(DynamicBatchConfig const &dynamicBatchConfig, std::ostream &os)
-
+static void serialize(RequestStatsPerIteration const &state, std::ostream &os)
-
-
-
-static size_t serializedSize(DynamicBatchConfig const &dynamicBatchConfig)
-
+static std::vector<char> serialize(RequestStatsPerIteration const &state)
-
-
-
-static SchedulerConfig deserializeSchedulerConfig(std::istream &is)
-
+static size_t serializedSize(RequestStatsPerIteration const &state)
-
-
-
-static void serialize(SchedulerConfig const &schedulerConfig, std::ostream &os)
-
+static std::vector<char> serialize(std::vector<RequestStatsPerIteration> const &requestStatsVec)
-
-
-
-static size_t serializedSize(SchedulerConfig const &schedulerConfig)
-
+static std::vector<RequestStatsPerIteration> deserializeRequestStatsPerIterationVec(std::vector<char> &buffer)
-
-
-
-static ExtendedRuntimePerfKnobConfig deserializeExtendedRuntimePerfKnobConfig(std::istream &is)
-
+static std::string deserializeString(std::istream &is)
-
-
-
-static void serialize(ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig, std::ostream &os)
-
+static bool deserializeBool(std::istream &is)
-
-
-
-static size_t serializedSize(ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig)
-
+static ModelType deserializeModelType(std::istream &is)
-
-
-
-static ParallelConfig deserializeParallelConfig(std::istream &is)
-
-static RequestPerfMetrics::TimePoint deserializeTimePoint(std::istream &is)
-
+
-
+namespace kv_cache
-
-
-
-static void serialize(ParallelConfig const ¶llelConfig, std::ostream &os)
-
-class Serialization
-
-namespace executor
-
+namespace tensorrt_llm
-
+
-
+namespace executor
-
+
-
+class Shape : public tensorrt_llm::common::ArrayView<detail::DimType64 const>
- ++
Public Types
+-
+
-
+using Base = tensorrt_llm::common::ArrayView<detail::DimType64 const>
-
-
-
-static size_t serializedSize(ParallelConfig const ¶llelConfig)
++Public Functions
-
-
-
-static PeftCacheConfig deserializePeftCacheConfig(std::istream &is)
-
+inline Shape()
-
-
-
-static void serialize(PeftCacheConfig const &peftCacheConfig, std::ostream &os)
-
+inline Shape(DimType64 const *data, Base::size_type size)
-
-
-
-static size_t serializedSize(PeftCacheConfig const &peftCacheConfig)
-
+inline Shape(std::initializer_list<DimType64> dims)
-
-
-
-static OrchestratorConfig deserializeOrchestratorConfig(std::istream &is)
-
+using Base = tensorrt_llm::common::ArrayView<detail::DimType64 const>
-
+
-
+class Tensor
- ++
Public Types
+-
+
-
+using CudaStreamPtr = std::shared_ptr<runtime::CudaStream>
++Public Functions
-
-
-
-static void serialize(OrchestratorConfig const &orchestratorConfig, std::ostream &os)
-
+Tensor copyToCpu(Tensor::CudaStreamPtr stream = nullptr) const
-
-
-
-static size_t serializedSize(OrchestratorConfig const &orchestratorConfig)
-
+Tensor copyToPinned(Tensor::CudaStreamPtr stream = nullptr) const
-
-
-
-static DecodingMode deserializeDecodingMode(std::istream &is)
-
+Tensor copyToPooledPinned(Tensor::CudaStreamPtr stream = nullptr) const
-
-
-
-static void serialize(DecodingMode const &decodingMode, std::ostream &os)
-
+Tensor copyToManaged(Tensor::CudaStreamPtr stream = nullptr) const
-
-
-
-static size_t serializedSize(DecodingMode const &decodingMode)
-
+Tensor copyToGpu(Tensor::CudaStreamPtr stream) const
-
-
-
-static LookaheadDecodingConfig deserializeLookaheadDecodingConfig(std::istream &is)
-
+Tensor() noexcept = default
-
-
-
-static void serialize(LookaheadDecodingConfig const &lookaheadDecodingConfig, std::ostream &os)
-
+~Tensor() = default
-
-
-
-static size_t serializedSize(LookaheadDecodingConfig const &lookaheadDecodingConfig)
-
+Tensor(Tensor const &other) noexcept = default
-
-
-
-static EagleConfig deserializeEagleConfig(std::istream &is)
-
+Tensor(Tensor &&other) noexcept = default
-
-
-
-static void serialize(EagleConfig const &eagleConfig, std::ostream &os)
-
+Tensor &operator=(Tensor const &other) noexcept = default
-
-
-
-static size_t serializedSize(EagleConfig const &eagleConfig)
-
+Tensor &operator=(Tensor &&other) noexcept = default
-
-
-
-static SpeculativeDecodingConfig deserializeSpeculativeDecodingConfig(std::istream &is)
- +void *getData()
+Returns a pointer to underlying array.
+-
-
-
-static void serialize(SpeculativeDecodingConfig const &specDecConfig, std::ostream &os)
- +void const *getData() const
+Returns a pointer to underlying array.
+-
-
-
-static size_t serializedSize(SpeculativeDecodingConfig const &specDecConfig)
- +DataType getDataType() const
+Returns the data type of the buffer.
+-
-
-
-static GuidedDecodingConfig deserializeGuidedDecodingConfig(std::istream &is)
- +MemoryType getMemoryType() const
+Returns the memory type of the buffer.
+-
-
-
-static void serialize(GuidedDecodingConfig const &guidedDecodingConfig, std::ostream &os)
- +Shape getShape() const
+Returns the tensor dimensions.
+-
-
-
-static size_t serializedSize(GuidedDecodingConfig const &guidedDecodingConfig)
- +std::size_t getSize() const
+Returns the number of elements in the tensor.
+-
-
-
-static GuidedDecodingParams deserializeGuidedDecodingParams(std::istream &is)
- +std::size_t getSizeInBytes() const
+Returns the size of the tensor in bytes.
+-
-
-
-static void serialize(GuidedDecodingParams const &guidedDecodingParams, std::ostream &os)
- +void setZero(CudaStreamPtr stream = nullptr)
+Set the entire memory to zero.
+-
+
- Parameters: +
stream – Must be a valid CUDA stream if the memory type is GPU.
+
+
-
-
-
-static size_t serializedSize(GuidedDecodingParams const &guidedDecodingParams)
- +void setFrom(Tensor const &other, CudaStreamPtr stream = nullptr)
+Copy the data and shape from another tensor.
+-
+
- Parameters: +
-
+
other – A tensor to copy from.
+stream – Must be a valid CUDA stream if the memory type is GPU.
+
+
-
-
-
-static KvCacheRetentionConfig deserializeKvCacheRetentionConfig(std::istream &is)
-
+inline explicit operator bool() const
-
-
-
-static void serialize(KvCacheRetentionConfig const &kvCacheRetentionConfig, std::ostream &os)
-
+inline bool operator==(Tensor const &rhs) const
-
-
-
-static size_t serializedSize(KvCacheRetentionConfig const &kvCacheRetentionConfig)
-
+inline bool operator!=(Tensor const &rhs) const
++ +Public Static Functions
-
-
-
-static KvCacheRetentionConfig::TokenRangeRetentionConfig deserializeTokenRangeRetentionConfig(std::istream &is)
- +static Tensor cpu(DataType dataType, Shape shape = {})
+Allocate a cpu tensor with the given shape and data type.
+-
+
- Parameters: +
-
+
shape – The shape of the tensor.
+dataType – The data type of the tensor.
+
+
-
-
-
-static void serialize(KvCacheRetentionConfig::TokenRangeRetentionConfig const &tokenRangeRetentionConfig, std::ostream &os)
-
+template<typename T>
static inline Tensor cpu(Shape shape = {})
-
-
-
-static size_t serializedSize(KvCacheRetentionConfig::TokenRangeRetentionConfig const &tokenRangeRetentionConfig)
- +static Tensor pinned(DataType dataType, Shape shape = {})
+Allocate a cpu tensor in pinned memory with the given shape and data type.
+-
+
- Parameters: +
-
+
shape – The shape of the tensor.
+dataType – The data type of the tensor.
+
+
-
-
-
-static DecodingConfig deserializeDecodingConfig(std::istream &is)
-
+template<typename T>
static inline Tensor pinned(Shape shape = {})
-
-
-
-static void serialize(DecodingConfig const &decodingConfig, std::ostream &os)
- +static Tensor pooledPinned(DataType dataType, Shape shape = {})
+Allocate a cpu tensor in pooled pinned memory with the given shape and data type.
+-
+
- Parameters: +
-
+
shape – The shape of the tensor.
+dataType – The data type of the tensor.
+
+
-
-
-
-static size_t serializedSize(DecodingConfig const &decodingConfig)
-
+template<typename T>
static inline Tensor pooledPinned(Shape shape = {})
-
-
-
-static DebugConfig deserializeDebugConfig(std::istream &is)
- +static Tensor managed(DataType dataType, Shape shape = {})
+Allocate a tensor in managed memory (UVM) with the given shape and data type.
+-
+
- Parameters: +
-
+
shape – The shape of the tensor.
+dataType – The data type of the tensor.
+
+
-
-
-
-static void serialize(DebugConfig const &debugConfig, std::ostream &os)
-
+template<typename T>
static inline Tensor managed(Shape shape = {})
-
-
-
-static size_t serializedSize(DebugConfig const &debugConfig)
- +static Tensor gpu(DataType dataType, CudaStreamPtr stream, Shape shape = {})
+Allocate a gpu tensor with the given shape and data type on a particular cuda stream.
+-
+
- Parameters: +
-
+
shape – The shape of the tensor.
+stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.
+dataType – The data type of the tensor.
+
+
-
-
-
-static ExecutorConfig deserializeExecutorConfig(std::istream &is)
-
+template<typename T>
static inline Tensor gpu(CudaStreamPtr stream, Shape shape = {})
-
-
-
-static void serialize(ExecutorConfig const &executorConfig, std::ostream &os)
- +static Tensor of(DataType dataType, void *data, Shape shape)
+Wrap a data pointer into a tensor without taking ownership.
+-
+
- Parameters: +
-
+
shape – The shape of the tensor.
+dataType – The data type of the tensor.
+stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.
+
+
-
-
-
-static size_t serializedSize(ExecutorConfig const &executorConfig)
- +template<typename T>
+
static inline Tensor of(T *data, Shape shape)Wrap a data pointer into a tensor without taking ownership.
+-
+
- Parameters: +
-
+
shape – The shape of the tensor.
+dataType – The data type of the tensor.
+stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.
+
+
-
-
-
-static KvCacheStats deserializeKvCacheStats(std::istream &is)
- +template<typename T>
+
static inline Tensor of(T &data)Wrap any container into a tensor without taking ownership.
+-
+
- Parameters: +
-
+
shape – The shape of the tensor.
+dataType – The data type of the tensor.
+stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.
+
+
-
-
-
-static void serialize(KvCacheStats const &kvCacheStats, std::ostream &os)
++ +Private Functions
-
-
-
-static size_t serializedSize(KvCacheStats const &kvCacheStats)
-
-
-
-static StaticBatchingStats deserializeStaticBatchingStats(std::istream &is)
-
-
-
-static void serialize(StaticBatchingStats const &staticBatchingStats, std::ostream &os)
++Private Static Functions
-
-
-
-static size_t serializedSize(StaticBatchingStats const &staticBatchingStats)
-
+template<typename T>
static inline DataType getRuntimeType()
-
-
-
-static InflightBatchingStats deserializeInflightBatchingStats(std::istream &is)
++Friends
+-
+
- +friend class Serialization +
-
-
-
-static void serialize(InflightBatchingStats const &inflightBatchingStats, std::ostream &os)
-
+friend std::shared_ptr<runtime::ITensor> const &toITensor(Tensor const &tensor)
-
-
-
-static size_t serializedSize(InflightBatchingStats const &inflightBatchingStats)
-
-
-
-static IterationStats deserializeIterationStats(std::vector<char> &buffer)
-
+using CudaStreamPtr = std::shared_ptr<runtime::CudaStream>
-
-
-
-static IterationStats deserializeIterationStats(std::istream &is)
-
+namespace detail
- ++
Typedefs
+-
+
-
+using DimType64 = int64_t
++Functions
-
-
-
-static void serialize(IterationStats const &iterStats, std::ostream &os)
-
+std::shared_ptr<runtime::ITensor> const &toITensor(Tensor const &tensor)
-
-
-
-static std::vector<char> serialize(IterationStats const &iterStats)
-
-
-
-static size_t serializedSize(IterationStats const &iterStats)
-
+using DimType64 = int64_t
-
-static std::vector<char> serialize(std::vector<IterationStats> const &iterStatsVec)
-
+
-
-
-
+class Shape : public tensorrt_llm::common::ArrayView<detail::DimType64 const>
-
-
-
-static std::vector<IterationStats> deserializeIterationStatsVec(std::vector<char> &buffer)
-
+namespace runtime
-
+
-
+namespace executor
-
+namespace tensorrt_llm
-
+
-
+namespace executor
-
+
-
+namespace disagg_executor
-
+
-
+class DisaggExecutorOrchestrator
- ++
Public Functions
-
-
-
-static DisServingRequestStats deserializeDisServingRequestStats(std::istream &is)
- +DisaggExecutorOrchestrator(std::vector<std::filesystem::path> const &ctxEnginePaths, std::vector<std::filesystem::path> const &genEnginePaths, std::vector<executor::ExecutorConfig> const &ctxExecutorConfigs, std::vector<executor::ExecutorConfig> const &genExecutorConfigs, bool hasContextAwaitThreads, bool hasGenAwaitThreads)
+Constructs a DisaggExecutorOrchestrator object.
+-
+
- Parameters: +
-
+
ctxEnginePaths – A vector of file paths to context engine files.
+genEnginePaths – A vector of file paths to generation engine files.
+ctxExecutorConfigs – A vector of ExecutorConfig for context executors.
+genExecutorConfigs – A vector of ExecutorConfig for generation executors.
+hasContextAwaitThreads – Whether or not there are threads that receive response for each generation executor.
+hasGenAwaitThreads – Whether or not there are threads that receive response for each generation executor.
+
+
-
-
-
-static void serialize(DisServingRequestStats const &stats, std::ostream &os)
- +std::vector<IdType> enqueueContext(std::vector<texec::Request> const &requests, std::optional<int> selectContextId = std::nullopt, bool batch = false)
+Enqueue context-only requests to context executors.
+-
+
- Parameters: +
-
+
requests – A vector of context-only requests.
+selectContextId – The index of the context executor to use. If
std::nullopt, the executor that has the smallest number of inflight requests will be used.
+batch – If true,enqueue requests in same context executor.If false, will try to use a different executor for each request.
+
+- Returns: +
A vector of global request ids, corresponding to the order of the requests in
+requests, the id returned may be different from the request id in each executor.
+
-
-
-
-static size_t serializedSize(DisServingRequestStats const &disServingRequestStats)
- +void enqueueGeneration(std::vector<texec::Request> const &requests, std::vector<IdType> const &globalRequestIds, std::optional<int> selectGenIdx = std::nullopt, bool batch = false)
+Enqueue generation-only requests to generation executors.
+-
+
- Parameters: +
-
+
requests – A vector of generation-only requests.
+globalRequestIds – A vector of global request ids, corresponding to the order of the requests,and must be the ids returned by the enqueueContext function.
+selectGenIdx – The index of the generation executor to use. If
std::nullopt, the executor that has the smallest number of inflight requests will be used.
+batch – If true,enqueue requests in same generation executor.If false, will try to use a different executor for each request.
+
+
-
-
-
-static RequestStage deserializeRequestStage(std::istream &is)
- +std::vector<ResponseWithId> awaitContextResponses(std::optional<std::chrono::milliseconds> const &timeout, std::optional<int> contextIdx = std::nullopt)
+Await for context responses.
+-
+
- Parameters: +
-
+
timeout – The maximum time to wait for new responses
+contextIdx – The index of the context executor to use. If
std::nullopt, return ready responses in all context executors,ifhasContextAwaitThreadsis true, then this parameter must be std::nullopt.
+
+- Returns: +
A vector of responses with corresponding global request ids
+
+
-
-
-
-static void serialize(RequestStage const &requestStage, std::ostream &os)
- +std::vector<ResponseWithId> awaitGenerationResponses(std::optional<std::chrono::milliseconds> const &timeout, std::optional<int> genIdx = std::nullopt)
+Await for generation responses.
+-
+
- Parameters: +
-
+
timeout – The maximum time to wait for new responses.
+genIdx – The index of the generation executor to use. If
std::nullopt, return ready responses in all generation executors,ifhasGenAwaitThreadsis true, then this parameter must be std::nullopt.
+
+- Returns: +
A vector of responses with corresponding global request ids.
+
+
-
-
-
-static size_t serializedSize(RequestStage const &requestStage)
- +bool canEnqueue() const
+Indicates if the current process is allowed to enqueueRequests.
+-
-
-
-static RequestStats deserializeRequestStats(std::istream &is)
- +std::vector<std::unique_ptr<texec::Executor>> const &getContextExecutors() const
+Get context executors.
+-
-
-
-static void serialize(RequestStats const &state, std::ostream &os)
- +std::vector<std::unique_ptr<texec::Executor>> const &getGenExecutors() const
+Get generation executors.
+-
-
-
-static size_t serializedSize(RequestStats const &state)
-
+~DisaggExecutorOrchestrator()
-
-
-
-static RequestStatsPerIteration deserializeRequestStatsPerIteration(std::istream &is)
++Private Members
+-
+
-
+std::unique_ptr<Impl> mImpl
-
-static DisServingRequestStats deserializeDisServingRequestStats(std::istream &is)
-
+
-
+struct ResponseWithId
- ++
Public Functions
-
-
-
-static RequestStatsPerIteration deserializeRequestStatsPerIteration(std::vector<char> &buffer)
-
+inline ResponseWithId(tensorrt_llm::executor::Response &&response, IdType gid)
-
-
-
-static void serialize(RequestStatsPerIteration const &state, std::ostream &os)
-
+inline ResponseWithId(tensorrt_llm::executor::Response const &response, IdType gid)
-
-
-
-static std::vector<char> serialize(RequestStatsPerIteration const &state)
-
+inline ResponseWithId(ResponseWithId &&other) noexcept
-
-
-
-static size_t serializedSize(RequestStatsPerIteration const &state)
-
+ResponseWithId(ResponseWithId const &other) = default
-
-
-
-static std::vector<char> serialize(std::vector<RequestStatsPerIteration> const &requestStatsVec)
-
+inline ResponseWithId &operator=(ResponseWithId &&other) noexcept
-
-
-
-static std::vector<RequestStatsPerIteration> deserializeRequestStatsPerIterationVec(std::vector<char> &buffer)
-
+inline ResponseWithId &operator=(ResponseWithId const &other)
-
-
-
-static bool deserializeBool(std::istream &is)
-
-static RequestStatsPerIteration deserializeRequestStatsPerIteration(std::vector<char> &buffer)
-
-
-
-namespace kv_cache
-
+class DisaggExecutorOrchestrator
-
+namespace disagg_executor
serialization.h -
+ -
+namespace executor
-
+
-
-
-
-
+ -serialization.h
+-
+
-
+
- +serialization.h
--
-
+ +tensor.h
+-
+
+ disaggServerUtil.h
+-
+
-
+struct SpeculativeDecodingFastLogitsInfo
- +class SamplingConfig
-
-explicit PeftCacheConfig(SizeType32 numHostModuleLayer = 0, SizeType32 numDeviceModuleLayer = 0, SizeType32 optimalAdapterSize = kDefaultOptimalAdapterSize, SizeType32 maxAdapterSize = kDefaultMaxAdapterSize, SizeType32 numPutWorkers = 1, SizeType32 numEnsureWorkers = 1, SizeType32 numCopyStreams = 1, SizeType32 maxPagesPerBlockHost = kDefaultMaxPagesPerBlockHost, SizeType32 maxPagesPerBlockDevice = kDefaultMaxPagesPerBlockDevice, std::optional<float> const &deviceCachePercent = std::nullopt, std::optional<size_t> const &hostCacheSize = std::nullopt)
-
+Request(VecTokens inputTokenIds, SizeType32 maxTokens, bool streaming = false, SamplingConfig const &samplingConfig = SamplingConfig(), OutputConfig const &outputConfig = OutputConfig(), std::optional<SizeType32> const &endId = std::nullopt, std::optional<SizeType32> const &padId = std::nullopt, std::optional<std::vector<SizeType32>> positionIds = std::nullopt, std::optional<std::list<VecTokens>> badWords = std::nullopt, std::optional<std::list<VecTokens>> stopWords = std::nullopt, std::optional<Tensor> embeddingBias = std::nullopt, std::optional<ExternalDraftTokensConfig> externalDraftTokensConfig = std::nullopt, std::optional<PromptTuningConfig> pTuningConfig = std::nullopt, std::optional<MropeConfig> mRopeConfig = std::nullopt, std::optional<LoraConfig> loraConfig = std::nullopt, std::optional<LookaheadDecodingConfig> lookaheadConfig = std::nullopt, std::optional<KvCacheRetentionConfig> kvCacheRetentionConfig = std::nullopt, std::optional<std::string> logitsPostProcessorName = std::nullopt, std::optional<VecTokens> encoderInputTokenIds = std::nullopt, std::optional<IdType> clientId = std::nullopt, bool returnAllGeneratedTokens = false, PriorityType priority = kDefaultPriority, RequestType type = RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION, std::optional<ContextPhaseParams> contextPhaseParams = std::nullopt, std::optional<Tensor> encoderInputFeatures = std::nullopt, std::optional<SizeType32> encoderOutputLength = std::nullopt, std::optional<Tensor> crossAttentionMask = std::nullopt, SizeType32 numReturnSequences = 1, std::optional<EagleConfig> eagleConfig = std::nullopt, std::optional<Tensor> skipCrossAttnBlocks = std::nullopt, std::optional<GuidedDecodingParams> guidedDecodingParams = std::nullopt, std::optional<MillisecondsType> allottedTimeMs = std::nullopt)
- +class OrchestratorConfig
-
-enumerator kJSON
-
-enum class GuideType
-
-explicit ContextPhaseParams(VecTokens firstGenTokens, RequestIdType reqId)
-
-LookaheadDecodingConfig(SizeType32 windowSize, SizeType32 ngramSize, SizeType32 verificationSetSize)
-
-namespace disagg_executor
-
-namespace executor
-
+explicit DecodingConfig(std::optional<DecodingMode> decodingMode = std::nullopt, std::optional<LookaheadDecodingConfig> lookaheadDecodingConfig = std::nullopt, std::optional<MedusaChoices> medusaChoices = std::nullopt, std::optional<EagleConfig> eagleConfig = std::nullopt)
-
+
-
-
-
+
-
+namespace batch_manager
-
+using SizeType32 = std::int32_t
-
+namespace executor