diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 9c5554d4e..ded9b5310 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -107,5 +107,6 @@ workspace = true [dev-dependencies] futures-util = { version = "0.3", default-features = false } +rstest = "0.26" tempfile = "3.21" tokio = { version = "1.47", features = ["macros", "rt"] } diff --git a/crates/common/src/data_converters.rs b/crates/common/src/data_converters.rs index e58f37dad..ede06ba1a 100644 --- a/crates/common/src/data_converters.rs +++ b/crates/common/src/data_converters.rs @@ -1,7 +1,14 @@ //! Contains traits for and default implementations of data converters, codecs, and other //! serialization related functionality. -use crate::protos::temporal::api::{common::v1::Payload, failure::v1::Failure}; +mod failure_converter; + +pub use failure_converter::{ + ActivityExecutionDecodeHint, ChildWorkflowExecutionDecodeHint, ChildWorkflowSignalDecodeHint, + ChildWorkflowStartDecodeHint, DefaultFailureConverter, FailureConverter, FailureDecodeHint, +}; + +use crate::protos::temporal::api::common::v1::Payload; use futures::{FutureExt, future::BoxFuture}; use std::{collections::HashMap, sync::Arc}; @@ -22,6 +29,7 @@ impl std::fmt::Debug for DataConverter { .finish_non_exhaustive() } } + impl DataConverter { /// Create a new DataConverter with the given payload converter, failure converter, and codec. pub fn new( @@ -105,6 +113,34 @@ impl DataConverter { &self.payload_converter } + /// Returns the failure converter component of this data converter. + pub fn failure_converter(&self) -> &(dyn FailureConverter + Send + Sync) { + self.failure_converter.as_ref() + } + + /// Decode a Temporal failure into a caller-facing Rust error surface. + pub fn to_error( + &self, + context: &SerializationContextData, + failure: crate::protos::temporal::api::failure::v1::Failure, + hint: H, + ) -> Result { + let normalized = + self.failure_converter + .to_error(failure, &self.payload_converter, context)?; + Ok(hint.adapt(normalized)) + } + + /// Encode a typed Rust error surface into a Temporal failure. + pub fn to_failure( + &self, + context: &SerializationContextData, + error: crate::error::OutgoingError, + ) -> crate::protos::temporal::api::failure::v1::Failure { + self.failure_converter + .to_failure(error, &self.payload_converter, context) + } + /// Returns the codec component of this data converter. pub fn codec(&self) -> &(dyn PayloadCodec + Send + Sync) { self.codec.as_ref() @@ -196,26 +232,6 @@ impl std::error::Error for PayloadConversionError { } } -/// Converts between Rust errors and Temporal [`Failure`] protobufs. -pub trait FailureConverter { - /// Convert an error into a Temporal failure protobuf. - fn to_failure( - &self, - error: Box, - payload_converter: &PayloadConverter, - context: &SerializationContextData, - ) -> Result; - - /// Convert a Temporal failure protobuf back into a Rust error. - fn to_error( - &self, - failure: Failure, - payload_converter: &PayloadConverter, - context: &SerializationContextData, - ) -> Result, PayloadConversionError>; -} -/// Default (currently unimplemented) failure converter. -pub struct DefaultFailureConverter; /// Encodes and decodes payloads, enabling encryption or compression. pub trait PayloadCodec { /// Encode payloads before they are sent to the server. @@ -307,6 +323,53 @@ pub trait TemporalDeserializable: Sized { } } +/// A codec-decoded set of payloads that can be deserialized later with to a user provided type. +#[derive(Clone, Debug)] +pub struct DecodablePayloads { + payloads: Vec, + payload_converter: PayloadConverter, + context: SerializationContextData, +} + +impl DecodablePayloads { + /// Create a new decodable payload set from raw payloads and the converter context needed to + /// deserialize them later. + pub fn new( + payloads: Vec, + payload_converter: PayloadConverter, + context: SerializationContextData, + ) -> Self { + Self { + payloads, + payload_converter, + context, + } + } + + /// Deserialize these payloads into a typed value using the stored payload converter. + pub fn deserialize( + &self, + ) -> Result { + self.payload_converter.from_payloads( + &SerializationContext { + data: &self.context, + converter: &self.payload_converter, + }, + self.payloads.clone(), + ) + } + + /// Returns the underlying payloads. + pub fn raw(&self) -> &[Payload] { + &self.payloads + } + + /// Consume this value and return the underlying payloads as a [`RawValue`]. + pub fn into_raw(self) -> RawValue { + RawValue::new(self.payloads) + } +} + /// An unconverted set of payloads, used when the caller wants to defer deserialization. #[derive(Clone, Debug, Default)] pub struct RawValue { @@ -672,24 +735,6 @@ impl Default for DataConverter { ) } } -impl FailureConverter for DefaultFailureConverter { - fn to_failure( - &self, - _: Box, - _: &PayloadConverter, - _: &SerializationContextData, - ) -> Result { - todo!() - } - fn to_error( - &self, - _: Failure, - _: &PayloadConverter, - _: &SerializationContextData, - ) -> Result, PayloadConversionError> { - todo!() - } -} impl PayloadCodec for DefaultPayloadCodec { fn encode( &self, @@ -866,4 +911,53 @@ mod tests { let args: MultiArgs2 = ("hello".to_string(), 42i32).into(); assert_eq!(args, MultiArgs2("hello".to_string(), 42)); } + + fn decodable_from_value(value: &T) -> DecodablePayloads { + let converter = PayloadConverter::default(); + let payloads = converter + .to_payloads( + &SerializationContext { + data: &SerializationContextData::Workflow, + converter: &converter, + }, + value, + ) + .unwrap(); + DecodablePayloads::new(payloads, converter, SerializationContextData::Workflow) + } + #[test] + fn decodable_payloads_roundtrip_string() { + let payloads = decodable_from_value(&"hello".to_string()); + + let result: String = payloads.deserialize().unwrap(); + + assert_eq!(result, "hello"); + } + + #[test] + fn decodable_payloads_roundtrip_option_string() { + let payloads = decodable_from_value(&Some("hello".to_string())); + + let result: Option = payloads.deserialize().unwrap(); + + assert_eq!(result, Some("hello".to_string())); + } + + #[test] + fn decodable_payloads_roundtrip_unit() { + let payloads = decodable_from_value(&()); + + let result: () = payloads.deserialize().unwrap(); + + assert_eq!(result, ()); + } + + #[test] + fn decodable_payloads_roundtrip_vec_string() { + let payloads = decodable_from_value(&vec!["hello".to_string(), "world".to_string()]); + + let result: Vec = payloads.deserialize().unwrap(); + + assert_eq!(result, vec!["hello".to_string(), "world".to_string()]); + } } diff --git a/crates/common/src/data_converters/failure_converter.rs b/crates/common/src/data_converters/failure_converter.rs new file mode 100644 index 000000000..c667a538e --- /dev/null +++ b/crates/common/src/data_converters/failure_converter.rs @@ -0,0 +1,1490 @@ +//! Failure conversion sits at the normalized boundary between Rust-side error surfaces and +//! Temporal's proto [`Failure`] transport object. +//! +//! - [`FailureConverter`] owns translation between proto [`Failure`] and the SDK's shared +//! normalized error model. +//! - encode-side call sites adapt caller-facing errors into [`OutgoingError`] before reaching this +//! module. +//! - decode-side call sites first normalize proto failures into [`IncomingError`], then +//! [`FailureDecodeHint`] implementations adapt that normalized value into the caller-facing error +//! type they expect. + +use super::{PayloadConversionError, PayloadConverter, SerializationContextData}; +use crate::{ + error::{ + ActivityExecutionError, ActivityFailureError, ApplicationFailure, CancelledError, + ChildWorkflowExecutionError, ChildWorkflowFailureError, ChildWorkflowSignalError, + ChildWorkflowSignalFailureError, ChildWorkflowStartError, IncomingError, + IncomingNexusHandlerError, IncomingNexusOperationExecutionError, OutgoingActivityError, + OutgoingError, OutgoingWorkflowError, ResetWorkflowError, ServerError, TerminatedError, + TimeoutError, + }, + protos::temporal::api::failure::v1::{ + ActivityFailureInfo, ApplicationFailureInfo, CanceledFailureInfo, + ChildWorkflowExecutionFailureInfo, Failure, failure::FailureInfo, + }, +}; + +/// Converts between Rust errors and Temporal [`Failure`] protobufs. +pub trait FailureConverter { + /// Convert an error into a Temporal failure protobuf. + fn to_failure( + &self, + error: OutgoingError, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Failure; + + /// Convert a Temporal failure protobuf back into a Rust error. + fn to_error( + &self, + failure: Failure, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Result; +} + +/// Default failure converter. +pub struct DefaultFailureConverter; + +/// Adapts a normalized incoming failure into a caller-facing error surface. +pub trait FailureDecodeHint { + /// The caller-facing error type produced by this hint. + type Output; + + /// Adapt a normalized incoming error to the caller-facing output. + fn adapt(self, normalized: IncomingError) -> Self::Output; +} + +/// Decode hint for activity execution results. +#[derive(Debug, Clone, Copy)] +pub struct ActivityExecutionDecodeHint { + /// Whether the workflow-side resolution was cancelled rather than failed. + pub cancelled: bool, +} + +impl FailureDecodeHint for ActivityExecutionDecodeHint { + type Output = ActivityExecutionError; + + fn adapt(self, normalized: IncomingError) -> Self::Output { + match normalized { + IncomingError::Activity(activity) => { + if self.cancelled && matches!(activity.cause(), Some(IncomingError::Cancelled(_))) { + // We collapse to the inner cancellation error so callers do not see a cancel + // caused by another cancel. + let (_, cause) = activity.into_parts(); + let Some(IncomingError::Cancelled(cancelled)) = cause else { + unreachable!("checked above"); + }; + ActivityExecutionError::Cancelled(cancelled) + } else { + ActivityExecutionError::Failed(activity) + } + } + other => match other { + IncomingError::Cancelled(cancelled) if self.cancelled => { + ActivityExecutionError::Cancelled(cancelled) + } + other => { + let activity = ActivityFailureError::new( + other.into_failure(), + ActivityFailureInfo::default(), + None, + ); + ActivityExecutionError::Failed(activity) + } + }, + } + } +} + +/// Decode hint for child-workflow start results. +#[derive(Debug, Clone, Copy)] +pub struct ChildWorkflowStartDecodeHint; + +impl FailureDecodeHint for ChildWorkflowStartDecodeHint { + type Output = ChildWorkflowStartError; + + fn adapt(self, normalized: IncomingError) -> Self::Output { + match normalized { + IncomingError::Cancelled(cancelled) => { + ChildWorkflowStartError::Cancelled(Box::new(cancelled)) + } + other => { + let payload_converter = PayloadConverter::default(); + ChildWorkflowStartError::Cancelled(Box::new(CancelledError::new( + other.into_failure(), + CanceledFailureInfo::default(), + None, + &payload_converter, + &SerializationContextData::None, + ))) + } + } + } +} + +/// Decode hint for child-workflow execution results. +#[derive(Debug, Clone, Copy)] +pub struct ChildWorkflowExecutionDecodeHint; + +impl FailureDecodeHint for ChildWorkflowExecutionDecodeHint { + type Output = ChildWorkflowExecutionError; + + fn adapt(self, normalized: IncomingError) -> Self::Output { + match normalized { + IncomingError::ChildWorkflowExecution(child) => { + ChildWorkflowExecutionError::Failed(Box::new(child)) + } + other => ChildWorkflowExecutionError::Failed(Box::new(ChildWorkflowFailureError::new( + other.into_failure(), + ChildWorkflowExecutionFailureInfo::default(), + None, + ))), + } + } +} + +/// Decode hint for child-workflow signal failures. +#[derive(Debug, Clone, Copy)] +pub struct ChildWorkflowSignalDecodeHint; + +impl FailureDecodeHint for ChildWorkflowSignalDecodeHint { + type Output = ChildWorkflowSignalError; + + fn adapt(self, normalized: IncomingError) -> Self::Output { + let failure = normalized.failure().clone(); + ChildWorkflowSignalError::Failed(Box::new(ChildWorkflowSignalFailureError::new( + failure, normalized, + ))) + } +} + +impl FailureConverter for DefaultFailureConverter { + fn to_failure( + &self, + error: OutgoingError, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Failure { + let original_error = error.to_string(); + let encoded = match error { + OutgoingError::Activity(activity) => { + encode_outgoing_activity_error(activity, payload_converter, context) + } + OutgoingError::Workflow(OutgoingWorkflowError::Application(app)) => { + app.encode_failure(payload_converter, context) + } + OutgoingError::Workflow(OutgoingWorkflowError::ActivityExecution(activity)) => { + activity.encode_failure(payload_converter, context) + } + OutgoingError::Workflow(OutgoingWorkflowError::ChildWorkflowExecution(child)) => { + child.encode_failure(payload_converter, context) + } + OutgoingError::Workflow(OutgoingWorkflowError::ChildWorkflowStart(child)) => { + child.encode_failure(payload_converter, context) + } + OutgoingError::Workflow(OutgoingWorkflowError::ChildWorkflowSignal(signal)) => { + signal.encode_failure(payload_converter, context) + } + }; + encoded.unwrap_or_else(|converter_error| { + Failure::application_failure( + failed_error_conversion_message(&original_error, &converter_error), + false, + ) + }) + } + + fn to_error( + &self, + failure: Failure, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Result { + Ok(decode_failure(failure, payload_converter, context)) + } +} + +/// Trait for expressing that a type has a known conversion to a Failure proto +trait EncodeFailure { + fn encode_failure( + &self, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Result; +} + +enum ClassifiedFailure<'a> { + Application(&'a ApplicationFailure), + ActivityExecution(&'a ActivityExecutionError), + ChildWorkflowExecution(&'a ChildWorkflowExecutionError), + ChildWorkflowStart(&'a ChildWorkflowStartError), + ChildWorkflowSignal(&'a ChildWorkflowSignalError), + Generic(&'a (dyn std::error::Error + 'static)), +} + +fn failed_error_conversion_message( + original_error: impl std::fmt::Display, + converter_error: &PayloadConversionError, +) -> String { + format!( + "Failed converting error to failure: {converter_error}, original error message: \ + {original_error}" + ) +} + +impl<'a> ClassifiedFailure<'a> { + fn from_error(err: &'a (dyn std::error::Error + 'static)) -> Self { + if let Some(app) = err.downcast_ref::() { + Self::Application(app) + } else if let Some(activity) = err.downcast_ref::() { + Self::ActivityExecution(activity) + } else if let Some(child) = err.downcast_ref::() { + Self::ChildWorkflowExecution(child) + } else if let Some(child) = err.downcast_ref::() { + Self::ChildWorkflowStart(child) + } else if let Some(child_signal) = err.downcast_ref::() { + Self::ChildWorkflowSignal(child_signal) + } else { + Self::Generic(err) + } + } + + fn encode(self) -> Failure { + match self { + Self::Application(app) => app + .encode_failure( + &PayloadConverter::default(), + &SerializationContextData::None, + ) + .unwrap_or_else(|converter_error| { + encode_failed_error_conversion(app, converter_error) + }), + Self::ActivityExecution(activity) => activity + .encode_failure( + &PayloadConverter::default(), + &SerializationContextData::None, + ) + .unwrap_or_else(|converter_error| { + encode_failed_error_conversion(activity, converter_error) + }), + Self::ChildWorkflowExecution(child) => child + .encode_failure( + &PayloadConverter::default(), + &SerializationContextData::None, + ) + .unwrap_or_else(|converter_error| { + encode_failed_error_conversion(child, converter_error) + }), + Self::ChildWorkflowStart(child) => child + .encode_failure( + &PayloadConverter::default(), + &SerializationContextData::None, + ) + .unwrap_or_else(|converter_error| { + encode_failed_error_conversion(child, converter_error) + }), + Self::ChildWorkflowSignal(signal) => signal + .encode_failure( + &PayloadConverter::default(), + &SerializationContextData::None, + ) + .unwrap_or_else(|converter_error| { + encode_failed_error_conversion(signal, converter_error) + }), + Self::Generic(err) => encode_generic_application_failure(err), + } + } +} + +impl EncodeFailure for ApplicationFailure { + fn encode_failure( + &self, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Result { + let details = self + .failure_payloads() + .map(|details| details.encode(payload_converter, context)) + .transpose()?; + Ok(Failure { + message: self.to_string(), + cause: self + .cause() + .map(|cause| Box::new(cause.failure().clone())) + .or_else(|| encode_application_failure_cause(self.source_error().as_ref())), + failure_info: Some(FailureInfo::ApplicationFailureInfo( + ApplicationFailureInfo { + r#type: self.type_name().unwrap_or_default().to_owned(), + non_retryable: self.is_non_retryable(), + details, + next_retry_delay: self.next_retry_delay().and_then(|d| d.try_into().ok()), + category: self.category() as i32, + }, + )), + ..Default::default() + }) + } +} + +fn encode_application_failure_cause( + source: &(dyn std::error::Error + 'static), +) -> Option> { + if matches!( + ClassifiedFailure::from_error(source), + ClassifiedFailure::Application(_) | ClassifiedFailure::Generic(_) + ) { + source.source().map(encode_error_as_failure).map(Box::new) + } else { + Some(Box::new(encode_error_as_failure(source))) + } +} + +fn encode_error_as_failure(err: &(dyn std::error::Error + 'static)) -> Failure { + ClassifiedFailure::from_error(err).encode() +} + +impl EncodeFailure for ActivityExecutionError { + fn encode_failure( + &self, + _: &PayloadConverter, + _: &SerializationContextData, + ) -> Result { + Ok(match self { + Self::Failed(failure) => failure.failure().clone(), + Self::Cancelled(failure) => failure.failure().clone(), + Self::Serialization(err) => encode_generic_application_failure(err), + }) + } +} + +impl EncodeFailure for ChildWorkflowExecutionError { + fn encode_failure( + &self, + _: &PayloadConverter, + _: &SerializationContextData, + ) -> Result { + Ok(match self { + Self::Failed(failure) => failure.failure().clone(), + Self::Serialization(_) => encode_generic_application_failure(self), + }) + } +} + +impl EncodeFailure for ChildWorkflowStartError { + fn encode_failure( + &self, + _: &PayloadConverter, + _: &SerializationContextData, + ) -> Result { + Ok(match self { + Self::Cancelled(failure) => failure.failure().clone(), + Self::StartFailed { .. } | Self::Serialization(_) => { + encode_generic_application_failure(self) + } + }) + } +} + +impl EncodeFailure for ChildWorkflowSignalError { + fn encode_failure( + &self, + _: &PayloadConverter, + _: &SerializationContextData, + ) -> Result { + Ok(match self { + Self::Failed(failure) => failure.failure().clone(), + Self::Serialization(err) => encode_generic_application_failure(err), + }) + } +} + +fn encode_outgoing_activity_error( + err: OutgoingActivityError, + payload_converter: &PayloadConverter, + context: &SerializationContextData, +) -> Result { + Ok(match err { + OutgoingActivityError::Application(app) => { + app.encode_failure(payload_converter, context)? + } + OutgoingActivityError::Cancelled { details } => Failure { + message: "Activity cancelled".to_string(), + failure_info: Some(FailureInfo::CanceledFailureInfo(CanceledFailureInfo { + details: details + .map(|details| details.encode(payload_converter, context)) + .transpose()?, + identity: Default::default(), + })), + ..Default::default() + }, + }) +} + +fn encode_generic_application_failure(err: &(dyn std::error::Error + 'static)) -> Failure { + Failure { + message: err.to_string(), + cause: err.source().map(encode_error_as_failure).map(Box::new), + failure_info: Some(FailureInfo::ApplicationFailureInfo( + ApplicationFailureInfo::default(), + )), + ..Default::default() + } +} + +fn encode_failed_error_conversion( + err: &(dyn std::error::Error + 'static), + converter_error: PayloadConversionError, +) -> Failure { + Failure { + message: failed_error_conversion_message(err, &converter_error), + cause: err.source().map(encode_error_as_failure).map(Box::new), + failure_info: Some(FailureInfo::ApplicationFailureInfo( + ApplicationFailureInfo::default(), + )), + ..Default::default() + } +} + +fn decode_failure( + failure: Failure, + payload_converter: &PayloadConverter, + context: &SerializationContextData, +) -> IncomingError { + let cause = failure + .cause + .clone() + .map(|cause| decode_failure(*cause, payload_converter, context)); + match failure.failure_info.clone() { + Some(FailureInfo::ApplicationFailureInfo(_)) | None => IncomingError::Application( + ApplicationFailure::from_failure(failure, cause, payload_converter, context), + ), + Some(FailureInfo::TimeoutFailureInfo(failure_info)) => IncomingError::Timeout( + TimeoutError::new(failure, failure_info, cause, payload_converter, context), + ), + Some(FailureInfo::CanceledFailureInfo(failure_info)) => IncomingError::Cancelled( + CancelledError::new(failure, failure_info, cause, payload_converter, context), + ), + Some(FailureInfo::TerminatedFailureInfo(_)) => { + IncomingError::Terminated(TerminatedError::new(failure, cause)) + } + Some(FailureInfo::ServerFailureInfo(_)) => { + IncomingError::Server(ServerError::new(failure, cause)) + } + Some(FailureInfo::ResetWorkflowFailureInfo(_)) => { + IncomingError::ResetWorkflow(ResetWorkflowError::new(failure, cause)) + } + Some(FailureInfo::ActivityFailureInfo(failure_info)) => { + IncomingError::Activity(ActivityFailureError::new(failure, failure_info, cause)) + } + Some(FailureInfo::ChildWorkflowExecutionFailureInfo(failure_info)) => { + IncomingError::ChildWorkflowExecution(ChildWorkflowFailureError::new( + failure, + failure_info, + cause, + )) + } + Some(FailureInfo::NexusOperationExecutionFailureInfo(_)) => { + IncomingError::NexusOperationExecution(IncomingNexusOperationExecutionError::new( + failure, cause, + )) + } + Some(FailureInfo::NexusHandlerFailureInfo(_)) => { + IncomingError::NexusHandler(IncomingNexusHandlerError::new(failure, cause)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + data_converters::{GenericPayloadConverter, SerializationContext}, + protos::temporal::api::{ + common::v1::{Payload, Payloads}, + enums::v1::ApplicationErrorCategory, + failure::v1::{ + ActivityFailureInfo, ChildWorkflowExecutionFailureInfo, NexusHandlerFailureInfo, + NexusOperationFailureInfo, ResetWorkflowFailureInfo, ServerFailureInfo, + TerminatedFailureInfo, TimeoutFailureInfo, failure::FailureInfo, + }, + }, + }; + use rstest::rstest; + use std::fmt; + + #[derive(Debug, Clone, Copy)] + enum IncomingKind { + Application, + Timeout, + Cancelled, + Terminated, + Server, + ResetWorkflow, + Activity, + ChildWorkflowExecution, + NexusOperationExecution, + NexusHandler, + } + + #[derive(Debug, Clone, Copy)] + enum ActivityExecutionKind { + Failed, + Cancelled, + } + + #[derive(Debug)] + struct TestError { + message: &'static str, + source: Option>, + } + + impl TestError { + fn new( + message: &'static str, + source: Option>, + ) -> Self { + Self { message, source } + } + } + + impl fmt::Display for TestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.message) + } + } + + impl std::error::Error for TestError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source + .as_deref() + .map(|source| source as &(dyn std::error::Error + 'static)) + } + } + + struct AlwaysFailsSerialize; + + impl serde::Serialize for AlwaysFailsSerialize { + fn serialize(&self, _serializer: S) -> Result { + Err(serde::ser::Error::custom("serialize boom")) + } + } + + fn assert_incoming_kind(decoded: &IncomingError, expected: IncomingKind) { + match expected { + IncomingKind::Application => assert!(matches!(decoded, IncomingError::Application(_))), + IncomingKind::Timeout => assert!(matches!(decoded, IncomingError::Timeout(_))), + IncomingKind::Cancelled => assert!(matches!(decoded, IncomingError::Cancelled(_))), + IncomingKind::Terminated => assert!(matches!(decoded, IncomingError::Terminated(_))), + IncomingKind::Server => assert!(matches!(decoded, IncomingError::Server(_))), + IncomingKind::ResetWorkflow => { + assert!(matches!(decoded, IncomingError::ResetWorkflow(_))) + } + IncomingKind::Activity => assert!(matches!(decoded, IncomingError::Activity(_))), + IncomingKind::ChildWorkflowExecution => { + assert!(matches!(decoded, IncomingError::ChildWorkflowExecution(_))) + } + IncomingKind::NexusOperationExecution => { + assert!(matches!(decoded, IncomingError::NexusOperationExecution(_))) + } + IncomingKind::NexusHandler => { + assert!(matches!(decoded, IncomingError::NexusHandler(_))) + } + } + } + + fn convert(err: OutgoingWorkflowError) -> Failure { + DefaultFailureConverter.to_failure( + OutgoingError::Workflow(err), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + } + + fn data_converter() -> crate::data_converters::DataConverter { + crate::data_converters::DataConverter::new( + PayloadConverter::default(), + DefaultFailureConverter, + crate::data_converters::DefaultPayloadCodec, + ) + } + + fn cancelled_failure(message: &str) -> Failure { + Failure { + message: message.to_owned(), + failure_info: Some(FailureInfo::CanceledFailureInfo( + CanceledFailureInfo::default(), + )), + ..Default::default() + } + } + + fn timeout_failure(message: &str) -> Failure { + Failure { + message: message.to_owned(), + failure_info: Some(FailureInfo::TimeoutFailureInfo( + TimeoutFailureInfo::default(), + )), + ..Default::default() + } + } + + #[test] + fn application_failures_preserve_metadata() { + let failure = convert(OutgoingWorkflowError::Application(Box::new( + ApplicationFailure::builder(anyhow::anyhow!("app boom")) + .type_name("MyType".to_owned()) + .non_retryable(true) + .category(ApplicationErrorCategory::Benign) + .details(crate::data_converters::RawValue::new(vec![Payload { + data: b"details".to_vec(), + ..Default::default() + }])) + .build(), + ))); + let Some(FailureInfo::ApplicationFailureInfo(info)) = failure.failure_info else { + panic!("expected application failure info"); + }; + assert_eq!(failure.message, "app boom"); + assert_eq!(info.r#type, "MyType"); + assert!(info.non_retryable); + assert_eq!(info.category(), ApplicationErrorCategory::Benign); + assert_eq!(info.details.unwrap().payloads[0].data, b"details".to_vec()); + } + + #[test] + fn application_failures_encode_serializable_details_with_payload_converter() { + let failure = convert(OutgoingWorkflowError::Application(Box::new( + ApplicationFailure::builder(anyhow::anyhow!("app boom")) + .details("detail") + .build(), + ))); + let Some(FailureInfo::ApplicationFailureInfo(info)) = failure.failure_info else { + panic!("expected application failure info"); + }; + let payloads = info.details.expect("details should be present").payloads; + let converter = PayloadConverter::default(); + let details: String = converter + .from_payloads( + &SerializationContext { + data: &SerializationContextData::Workflow, + converter: &converter, + }, + payloads, + ) + .unwrap(); + assert_eq!(details, "detail"); + } + + #[test] + fn application_failures_surface_detail_encoding_errors_with_original_message() { + let failure = DefaultFailureConverter.to_failure( + OutgoingError::Workflow(OutgoingWorkflowError::Application(Box::new( + ApplicationFailure::builder(anyhow::anyhow!("app boom")) + .details(AlwaysFailsSerialize) + .build(), + ))), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ); + + assert_eq!( + failure.message, + "Failed converting error to failure: Encoding error: serialize boom, original error message: app boom" + ); + } + + #[test] + fn application_failures_decode_details_through_payload_converter() { + let converter = PayloadConverter::default(); + let payloads = converter + .to_payloads( + &SerializationContext { + data: &SerializationContextData::Workflow, + converter: &converter, + }, + &"detail", + ) + .unwrap(); + let failure = Failure { + message: "app boom".to_owned(), + failure_info: Some(FailureInfo::ApplicationFailureInfo( + ApplicationFailureInfo { + details: Some(Payloads { payloads }), + ..Default::default() + }, + )), + ..Default::default() + }; + + let decoded = DefaultFailureConverter + .to_error(failure, &converter, &SerializationContextData::Workflow) + .unwrap(); + + let IncomingError::Application(app) = decoded else { + panic!("expected application error"); + }; + assert_eq!(app.details::().unwrap(), Some("detail".to_string())); + } + + #[test] + fn nested_application_failures_surface_detail_encoding_errors_in_fallback_failure() { + let app = ApplicationFailure::new(anyhow::Error::new(TestError::new( + "outer wrapper", + Some(Box::new( + ApplicationFailure::builder(anyhow::anyhow!("inner boom")) + .details(AlwaysFailsSerialize) + .build(), + )), + ))); + + let converted = convert(OutgoingWorkflowError::Application(Box::new(app))); + let cause = converted.cause.expect("expected nested fallback failure"); + assert_eq!( + cause.message, + "Failed converting error to failure: Encoding error: serialize boom, original error message: inner boom" + ); + assert!(matches!( + cause.failure_info, + Some(FailureInfo::ApplicationFailureInfo(_)) + )); + } + + #[test] + fn application_failures_do_not_duplicate_their_source_as_cause() { + let failure = convert(OutgoingWorkflowError::Application(Box::new( + ApplicationFailure::new(anyhow::anyhow!("app boom")), + ))); + + assert_eq!(failure.message, "app boom"); + assert!(failure.cause.is_none()); + } + + #[test] + fn application_failures_keep_special_causes_nested() { + let activity_failure = Failure { + message: "activity failed".to_owned(), + failure_info: Some(FailureInfo::ActivityFailureInfo( + ActivityFailureInfo::default(), + )), + ..Default::default() + }; + let app = ApplicationFailure::new(anyhow::Error::new(ActivityExecutionError::Failed( + ActivityFailureError::new( + activity_failure.clone(), + ActivityFailureInfo::default(), + None, + ), + ))); + let converted = convert(OutgoingWorkflowError::Application(Box::new(app))); + assert!(matches!( + converted.failure_info, + Some(FailureInfo::ApplicationFailureInfo(_)) + )); + assert_eq!(converted.cause.unwrap().as_ref(), &activity_failure); + } + + #[test] + fn application_failures_fall_back_to_source_error() { + let activity_failure = Failure { + message: "activity failed".to_owned(), + failure_info: Some(FailureInfo::ActivityFailureInfo( + ActivityFailureInfo::default(), + )), + ..Default::default() + }; + let app = ApplicationFailure::new(anyhow::Error::new(ActivityExecutionError::Failed( + ActivityFailureError::new( + activity_failure.clone(), + ActivityFailureInfo::default(), + None, + ), + ))); + + assert!(app.cause().is_none()); + + let converted = convert(OutgoingWorkflowError::Application(Box::new(app))); + + assert_eq!(converted.cause.unwrap().as_ref(), &activity_failure); + } + + #[test] + fn application_failures_skip_generic_wrappers_around_known_causes() { + let activity_failure = Failure { + message: "activity failed".to_owned(), + failure_info: Some(FailureInfo::ActivityFailureInfo( + ActivityFailureInfo::default(), + )), + ..Default::default() + }; + let app = ApplicationFailure::new(anyhow::Error::new(TestError::new( + "outer wrapper", + Some(Box::new(ActivityExecutionError::Failed( + ActivityFailureError::new( + activity_failure.clone(), + ActivityFailureInfo::default(), + None, + ), + ))), + ))); + + let converted = convert(OutgoingWorkflowError::Application(Box::new(app))); + + assert!(matches!( + converted.failure_info, + Some(FailureInfo::ApplicationFailureInfo(_)) + )); + assert_eq!(converted.message, "outer wrapper"); + assert_eq!(converted.cause.unwrap().as_ref(), &activity_failure); + } + + #[test] + fn application_failures_serialize_unknown_nested_causes_as_application_failures() { + let app = ApplicationFailure::new(anyhow::Error::new(TestError::new( + "outer wrapper", + Some(Box::new(TestError::new("generic inner cause", None))), + ))); + + let converted = convert(OutgoingWorkflowError::Application(Box::new(app))); + + assert!(matches!( + converted.failure_info, + Some(FailureInfo::ApplicationFailureInfo(_)) + )); + assert_eq!(converted.message, "outer wrapper",); + let cause = converted + .cause + .clone() + .expect("expected nested generic cause"); + assert_eq!(cause.message, "generic inner cause"); + assert!(matches!( + cause.failure_info, + Some(FailureInfo::ApplicationFailureInfo(_)) + )); + assert!(cause.cause.is_none()); + + let decoded = DefaultFailureConverter + .to_error( + converted.clone(), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + .unwrap(); + + let IncomingError::Application(decoded_app) = decoded else { + panic!("expected application error"); + }; + assert_eq!(decoded_app.failure(), Some(&converted)); + let Some(IncomingError::Application(wrapper)) = decoded_app.cause() else { + panic!("expected application cause"); + }; + assert_eq!( + wrapper.failure().map(|failure| failure.message.as_str()), + Some("generic inner cause") + ); + assert!(wrapper.cause().is_none()); + } + + #[test] + fn start_failed_child_workflow_errors_fall_back_to_application_failures() { + let failure = convert(OutgoingWorkflowError::ChildWorkflowStart(Box::new( + ChildWorkflowStartError::StartFailed { + workflow_id: "wf-id".to_owned(), + workflow_type: "wf-type".to_owned(), + cause: crate::protos::coresdk::child_workflow::StartChildWorkflowExecutionFailedCause::WorkflowAlreadyExists, + }, + ))); + assert!(matches!( + failure.failure_info, + Some(FailureInfo::ApplicationFailureInfo(_)) + )); + assert!(failure.message.contains("Child workflow start failed")); + } + + #[test] + fn application_failures_decode_with_metadata_and_proto() { + let failure = Failure { + message: "app boom".to_owned(), + failure_info: Some(FailureInfo::ApplicationFailureInfo( + ApplicationFailureInfo { + r#type: "MyType".to_owned(), + non_retryable: true, + ..Default::default() + }, + )), + ..Default::default() + }; + + let decoded = DefaultFailureConverter + .to_error( + failure.clone(), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + .unwrap(); + + let IncomingError::Application(app) = decoded else { + panic!("expected application error"); + }; + assert_eq!(app.type_name(), Some("MyType")); + assert!(app.is_non_retryable()); + assert_eq!(app.failure(), Some(&failure)); + } + + #[test] + fn application_failures_decode_with_normalized_cause() { + let failure = Failure { + message: "app boom".to_owned(), + cause: Some(Box::new(Failure { + message: "timed out".to_owned(), + failure_info: Some(FailureInfo::TimeoutFailureInfo( + TimeoutFailureInfo::default(), + )), + ..Default::default() + })), + failure_info: Some(FailureInfo::ApplicationFailureInfo( + ApplicationFailureInfo::default(), + )), + ..Default::default() + }; + + let decoded = DefaultFailureConverter + .to_error( + failure.clone(), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + .unwrap(); + + let IncomingError::Application(app) = decoded else { + panic!("expected application error"); + }; + assert_eq!(app.failure(), Some(&failure)); + assert!(matches!(app.cause(), Some(IncomingError::Timeout(_)))); + } + + #[test] + fn decoded_application_failures_preserve_cause() { + let failure = Failure { + message: "app boom".to_owned(), + cause: Some(Box::new(timeout_failure("timed out"))), + failure_info: Some(FailureInfo::ApplicationFailureInfo( + ApplicationFailureInfo::default(), + )), + ..Default::default() + }; + + let decoded = DefaultFailureConverter + .to_error( + failure.clone(), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + .unwrap(); + + let IncomingError::Application(app) = decoded else { + panic!("expected application error"); + }; + assert!(app.as_timeout().is_some()); + + let reencoded = convert(OutgoingWorkflowError::Application(Box::new(app))); + + assert_eq!(reencoded.message, failure.message); + assert_eq!(reencoded.cause.as_deref(), failure.cause.as_deref()); + + let decoded_reencoded = DefaultFailureConverter + .to_error( + reencoded, + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + .unwrap(); + let IncomingError::Application(roundtripped) = decoded_reencoded else { + panic!("expected application error"); + }; + assert!(roundtripped.as_timeout().is_some()); + } + + #[test] + fn application_failures_decode_wrapped_known_causes_without_collapsing_wrapper() { + let failure = Failure { + message: "app boom".to_owned(), + cause: Some(Box::new(Failure { + message: "activity failed".to_owned(), + failure_info: Some(FailureInfo::ActivityFailureInfo( + ActivityFailureInfo::default(), + )), + ..Default::default() + })), + failure_info: Some(FailureInfo::ApplicationFailureInfo( + ApplicationFailureInfo::default(), + )), + ..Default::default() + }; + + let decoded = DefaultFailureConverter + .to_error( + failure.clone(), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + .unwrap(); + + let IncomingError::Application(app) = decoded else { + panic!("expected application error"); + }; + assert_eq!(app.failure(), Some(&failure)); + let Some(IncomingError::Activity(activity)) = app.cause() else { + panic!("expected activity cause"); + }; + assert_eq!(activity.failure().message, "activity failed"); + } + + #[rstest] + #[case( + FailureInfo::ApplicationFailureInfo(ApplicationFailureInfo::default()), + IncomingKind::Application + )] + #[case( + FailureInfo::TimeoutFailureInfo(TimeoutFailureInfo::default()), + IncomingKind::Timeout + )] + #[case( + FailureInfo::CanceledFailureInfo(CanceledFailureInfo::default()), + IncomingKind::Cancelled + )] + #[case( + FailureInfo::TerminatedFailureInfo(TerminatedFailureInfo::default()), + IncomingKind::Terminated + )] + #[case( + FailureInfo::ServerFailureInfo(ServerFailureInfo::default()), + IncomingKind::Server + )] + #[case( + FailureInfo::ResetWorkflowFailureInfo(ResetWorkflowFailureInfo::default()), + IncomingKind::ResetWorkflow + )] + #[case( + FailureInfo::ActivityFailureInfo(ActivityFailureInfo::default()), + IncomingKind::Activity + )] + #[case( + FailureInfo::ChildWorkflowExecutionFailureInfo( + ChildWorkflowExecutionFailureInfo::default() + ), + IncomingKind::ChildWorkflowExecution + )] + #[case( + FailureInfo::NexusOperationExecutionFailureInfo(NexusOperationFailureInfo::default()), + IncomingKind::NexusOperationExecution + )] + #[case( + FailureInfo::NexusHandlerFailureInfo(NexusHandlerFailureInfo::default()), + IncomingKind::NexusHandler + )] + fn failure_info_decodes_to_expected_incoming_error( + #[case] failure_info: FailureInfo, + #[case] expected: IncomingKind, + ) { + let failure = Failure { + message: "boom".to_owned(), + failure_info: Some(failure_info), + ..Default::default() + }; + + let decoded = DefaultFailureConverter + .to_error( + failure.clone(), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + .unwrap(); + + assert_incoming_kind(&decoded, expected); + assert_eq!(decoded.failure(), &failure); + } + + #[test] + fn activity_decode_hint_preserves_timeout_reason() { + let failure = Failure { + message: "activity failed".to_owned(), + cause: Some(Box::new(Failure { + message: "timed out".to_owned(), + failure_info: Some(FailureInfo::TimeoutFailureInfo( + TimeoutFailureInfo::default(), + )), + ..Default::default() + })), + failure_info: Some(FailureInfo::ActivityFailureInfo(ActivityFailureInfo { + activity_id: "act-1".to_owned(), + activity_type: Some(crate::protos::temporal::api::common::v1::ActivityType { + name: "test-activity".to_owned(), + }), + scheduled_event_id: 5, + started_event_id: 6, + identity: "worker-1".to_owned(), + retry_state: crate::protos::temporal::api::enums::v1::RetryState::Timeout.into(), + })), + ..Default::default() + }; + let data_converter = crate::data_converters::DataConverter::new( + PayloadConverter::default(), + DefaultFailureConverter, + crate::data_converters::DefaultPayloadCodec, + ); + + let decoded = data_converter + .to_error( + &SerializationContextData::Workflow, + failure.clone(), + ActivityExecutionDecodeHint { cancelled: false }, + ) + .unwrap(); + + let ActivityExecutionError::Failed(decoded_failure) = decoded else { + panic!("expected failed activity execution error"); + }; + assert_eq!(decoded_failure.failure(), &failure); + assert_eq!(decoded_failure.activity_id(), "act-1"); + assert_eq!( + decoded_failure.activity_type().map(|ty| ty.name.as_str()), + Some("test-activity") + ); + assert_eq!(decoded_failure.scheduled_event_id(), 5); + assert_eq!(decoded_failure.started_event_id(), 6); + assert_eq!(decoded_failure.identity(), "worker-1"); + assert_eq!( + decoded_failure.retry_state(), + crate::protos::temporal::api::enums::v1::RetryState::Timeout + ); + assert!(matches!( + decoded_failure.cause(), + Some(IncomingError::Timeout(_)) + )); + } + + #[rstest] + #[case( + cancelled_failure("activity cancelled"), + ActivityExecutionKind::Cancelled, + None + )] + #[case(timeout_failure("timed out"), ActivityExecutionKind::Failed, None)] + #[case( + Failure { + message: "activity task cancelled".to_owned(), + cause: Some(Box::new(cancelled_failure("activity cancelled"))), + failure_info: Some(FailureInfo::ActivityFailureInfo( + ActivityFailureInfo::default(), + )), + ..Default::default() + }, + ActivityExecutionKind::Cancelled, + Some(cancelled_failure("activity cancelled")) + )] + fn activity_cancelled_decode_hint_adapts_expected_shape( + #[case] failure: Failure, + #[case] expected_kind: ActivityExecutionKind, + #[case] expected_failure: Option, + ) { + let decoded = data_converter() + .to_error( + &SerializationContextData::Workflow, + failure.clone(), + ActivityExecutionDecodeHint { cancelled: true }, + ) + .unwrap(); + + match expected_kind { + ActivityExecutionKind::Failed => { + let ActivityExecutionError::Failed(decoded_failure) = decoded else { + panic!("expected failed activity execution error"); + }; + assert_eq!(decoded_failure.failure(), &failure); + assert!(decoded_failure.cause().is_none()); + } + ActivityExecutionKind::Cancelled => { + let ActivityExecutionError::Cancelled(decoded_failure) = decoded else { + panic!("expected cancelled activity execution error"); + }; + assert_eq!( + decoded_failure.failure(), + expected_failure.as_ref().unwrap_or(&failure) + ); + assert!(decoded_failure.cause().is_none()); + } + } + } + + #[test] + fn timeout_error_exposes_timeout_info_fields() { + let heartbeat_details = crate::protos::temporal::api::common::v1::Payloads { + payloads: vec![Payload { + data: b"hb".to_vec(), + ..Default::default() + }], + }; + let failure = Failure { + message: "timed out".to_owned(), + failure_info: Some(FailureInfo::TimeoutFailureInfo(TimeoutFailureInfo { + timeout_type: crate::protos::temporal::api::enums::v1::TimeoutType::Heartbeat + .into(), + last_heartbeat_details: Some(heartbeat_details.clone()), + })), + ..Default::default() + }; + + let decoded = DefaultFailureConverter + .to_error( + failure.clone(), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + .unwrap(); + + let IncomingError::Timeout(timeout) = decoded else { + panic!("expected timeout error"); + }; + assert_eq!( + timeout.timeout_type(), + crate::protos::temporal::api::enums::v1::TimeoutType::Heartbeat + ); + assert_eq!( + timeout.raw_last_heartbeat_details(), + Some(heartbeat_details.payloads.as_slice()) + ); + assert_eq!(timeout.failure(), &failure); + } + + #[test] + fn cancelled_error_exposes_details() { + let details = crate::protos::temporal::api::common::v1::Payloads { + payloads: vec![Payload { + data: b"cancel".to_vec(), + ..Default::default() + }], + }; + let failure = Failure { + message: "cancelled".to_owned(), + failure_info: Some(FailureInfo::CanceledFailureInfo(CanceledFailureInfo { + details: Some(details.clone()), + identity: Default::default(), + })), + ..Default::default() + }; + + let decoded = DefaultFailureConverter + .to_error( + failure.clone(), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ) + .unwrap(); + + let IncomingError::Cancelled(cancelled) = decoded else { + panic!("expected cancelled error"); + }; + assert_eq!(cancelled.raw_details(), Some(details.payloads.as_slice())); + assert_eq!(cancelled.failure(), &failure); + } + + #[test] + fn child_workflow_decode_hint_preserves_child_failure_proto() { + let failure = Failure { + message: "child workflow failed".to_owned(), + failure_info: Some(FailureInfo::ChildWorkflowExecutionFailureInfo( + ChildWorkflowExecutionFailureInfo { + namespace: "default".to_owned(), + workflow_execution: Some( + crate::protos::temporal::api::common::v1::WorkflowExecution { + workflow_id: "child-id".to_owned(), + run_id: "run-id".to_owned(), + }, + ), + workflow_type: Some(crate::protos::temporal::api::common::v1::WorkflowType { + name: "child-type".to_owned(), + }), + initiated_event_id: 11, + started_event_id: 22, + retry_state: crate::protos::temporal::api::enums::v1::RetryState::Timeout + .into(), + }, + )), + ..Default::default() + }; + let decoded = data_converter() + .to_error( + &SerializationContextData::Workflow, + failure.clone(), + ChildWorkflowExecutionDecodeHint, + ) + .unwrap(); + + let ChildWorkflowExecutionError::Failed(decoded_failure) = decoded else { + panic!("expected failed child-workflow execution error"); + }; + assert_eq!(decoded_failure.failure(), &failure); + assert_eq!(decoded_failure.namespace(), "default"); + assert_eq!( + decoded_failure + .workflow_execution() + .map(|wf| wf.workflow_id.as_str()), + Some("child-id") + ); + assert_eq!( + decoded_failure + .workflow_execution() + .map(|wf| wf.run_id.as_str()), + Some("run-id") + ); + assert_eq!( + decoded_failure.workflow_type().map(|wf| wf.name.as_str()), + Some("child-type") + ); + assert_eq!(decoded_failure.initiated_event_id(), 11); + assert_eq!(decoded_failure.started_event_id(), 22); + assert_eq!( + decoded_failure.retry_state(), + crate::protos::temporal::api::enums::v1::RetryState::Timeout + ); + } + + #[rstest] + #[case( + Failure { + message: "child workflow cancelled".to_owned(), + cause: Some(Box::new(cancelled_failure("child workflow cancelled"))), + failure_info: Some(FailureInfo::ChildWorkflowExecutionFailureInfo( + ChildWorkflowExecutionFailureInfo::default(), + )), + ..Default::default() + }, + Some(IncomingKind::Cancelled) + )] + #[case(timeout_failure("timed out"), None)] + fn child_workflow_execution_decode_hint_adapts_expected_cause( + #[case] failure: Failure, + #[case] expected_cause: Option, + ) { + let decoded = data_converter() + .to_error( + &SerializationContextData::Workflow, + failure.clone(), + ChildWorkflowExecutionDecodeHint, + ) + .unwrap(); + + let ChildWorkflowExecutionError::Failed(decoded_failure) = decoded else { + panic!("expected failed child-workflow execution error"); + }; + assert_eq!(decoded_failure.failure(), &failure); + match expected_cause { + Some(expected) => { + let cause = decoded_failure + .cause() + .expect("expected child failure cause"); + assert_incoming_kind(cause, expected); + } + None => assert!(decoded_failure.cause().is_none()), + } + } + + #[test] + fn child_workflow_start_decode_hint_preserves_top_level_cancellation() { + let failure = Failure { + message: "child start cancelled".to_owned(), + failure_info: Some(FailureInfo::CanceledFailureInfo( + CanceledFailureInfo::default(), + )), + ..Default::default() + }; + let decoded = data_converter() + .to_error( + &SerializationContextData::Workflow, + failure.clone(), + ChildWorkflowStartDecodeHint, + ) + .unwrap(); + + let ChildWorkflowStartError::Cancelled(decoded_failure) = decoded else { + panic!("expected cancelled child-workflow start error"); + }; + assert_eq!(decoded_failure.failure(), &failure); + assert!(decoded_failure.cause().is_none()); + } + + #[test] + fn child_workflow_signal_decode_hint_preserves_failure_proto() { + let failure = Failure { + message: "child workflow signal failed".to_owned(), + cause: Some(Box::new(Failure { + message: "timed out".to_owned(), + failure_info: Some(FailureInfo::TimeoutFailureInfo( + TimeoutFailureInfo::default(), + )), + ..Default::default() + })), + ..Default::default() + }; + let decoded = data_converter() + .to_error( + &SerializationContextData::Workflow, + failure.clone(), + ChildWorkflowSignalDecodeHint, + ) + .unwrap(); + + let ChildWorkflowSignalError::Failed(decoded_failure) = decoded else { + panic!("expected failed child-workflow signal error"); + }; + assert_eq!(decoded_failure.failure(), &failure); + assert!(matches!( + decoded_failure.error(), + IncomingError::Application(_) + )); + assert!(matches!( + decoded_failure.cause(), + Some(IncomingError::Timeout(_)) + )); + assert!(std::error::Error::source(&decoded_failure).is_some()); + } + + #[test] + fn outgoing_cancelled_activity_errors_encode_to_cancelled_failures() { + let failure = DefaultFailureConverter.to_failure( + OutgoingError::Activity(OutgoingActivityError::Cancelled { details: None }), + &PayloadConverter::default(), + &SerializationContextData::Activity, + ); + + assert_eq!(failure.message, "Activity cancelled"); + assert!(matches!( + failure.failure_info, + Some(FailureInfo::CanceledFailureInfo(_)) + )); + } + + #[test] + fn outgoing_cancelled_activity_errors_encode_serializable_details_with_payload_converter() { + let failure = DefaultFailureConverter.to_failure( + OutgoingError::Activity(OutgoingActivityError::Cancelled { + details: Some("detail".to_string().into()), + }), + &PayloadConverter::default(), + &SerializationContextData::Activity, + ); + + let err = DefaultFailureConverter + .to_error( + failure, + &PayloadConverter::default(), + &SerializationContextData::Activity, + ) + .unwrap(); + let cancelled = err.as_cancelled().unwrap(); + let details: String = cancelled.details().unwrap().unwrap(); + assert_eq!(details, "detail"); + } +} diff --git a/crates/common/src/error.rs b/crates/common/src/error.rs new file mode 100644 index 000000000..b281bc2c3 --- /dev/null +++ b/crates/common/src/error.rs @@ -0,0 +1,1230 @@ +//! Shared error types used across Temporal SDK crates. + +use crate::{ + data_converters::{ + DecodablePayloads, GenericPayloadConverter, PayloadConversionError, PayloadConverter, + RawValue, SerializationContext, SerializationContextData, TemporalDeserializable, + TemporalSerializable, + }, + protos::{ + coresdk::child_workflow::StartChildWorkflowExecutionFailedCause, + temporal::api::{ + common::v1::{Payload, Payloads}, + enums::v1::{ApplicationErrorCategory, TimeoutType}, + failure::v1::Failure, + }, + }, +}; +use std::time::Duration; + +// We cannot store `Box` directly here because erased values still need +// to be driven back through the active `PayloadConverter` to reach serde-based implementations. +trait SerializableFailurePayload: Send + Sync { + fn to_payloads( + &self, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Result, PayloadConversionError>; +} + +impl SerializableFailurePayload for T +where + T: TemporalSerializable + Send + Sync + 'static, +{ + fn to_payloads( + &self, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Result, PayloadConversionError> { + payload_converter.to_payloads( + &SerializationContext { + data: context, + converter: payload_converter, + }, + self, + ) + } +} + +/// Payloads attached to a failure, either as a deferred outbound value or decoded inbound payloads. +#[derive(derive_more::Debug)] +pub struct FailurePayloads { + repr: FailurePayloadsRepr, +} + +#[derive(derive_more::Debug)] +enum FailurePayloadsRepr { + #[debug("Serializable(...)")] + Serializable(#[debug(skip)] Box), + Decoded(DecodablePayloads), +} + +impl FailurePayloads { + pub(crate) fn encode( + &self, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Result { + let payloads = match &self.repr { + FailurePayloadsRepr::Serializable(value) => { + value.to_payloads(payload_converter, context)? + } + FailurePayloadsRepr::Decoded(value) => value.raw().to_vec(), + }; + Ok(Payloads { payloads }) + } + + /// Deserialize the decoded payloads into a typed value. + pub fn deserialize( + &self, + ) -> Result { + match &self.repr { + FailurePayloadsRepr::Decoded(value) => value.deserialize(), + FailurePayloadsRepr::Serializable(_) => Err(PayloadConversionError::WrongEncoding), + } + } + + /// Returns the decoded raw payloads, if present. + pub fn raw(&self) -> Option<&[Payload]> { + match &self.repr { + FailurePayloadsRepr::Decoded(value) => Some(value.raw()), + FailurePayloadsRepr::Serializable(_) => None, + } + } + + /// Consume this value and return the decoded payloads as a [`RawValue`], if present. + pub fn into_raw(self) -> Option { + match self.repr { + FailurePayloadsRepr::Decoded(value) => Some(value.into_raw()), + FailurePayloadsRepr::Serializable(_) => None, + } + } +} + +impl From for FailurePayloads { + fn from(value: DecodablePayloads) -> Self { + Self { + repr: FailurePayloadsRepr::Decoded(value), + } + } +} + +impl From for FailurePayloads +where + T: TemporalSerializable + Send + Sync + 'static, +{ + fn from(value: T) -> Self { + Self { + repr: FailurePayloadsRepr::Serializable(Box::new(value)), + } + } +} + +/// User-authored application failure metadata that can be converted into a Temporal failure. +#[derive(Debug, bon::Builder)] +#[builder(start_fn = builder, state_mod(vis = "pub"))] +pub struct ApplicationFailure { + #[builder(start_fn, into)] + source: anyhow::Error, + type_name: Option, + #[builder(default)] + non_retryable: bool, + next_retry_delay: Option, + #[builder(default = ApplicationErrorCategory::Unspecified)] + category: ApplicationErrorCategory, + #[builder(into)] + details: Option, + failure: Option, + cause: Option>, +} + +impl ApplicationFailure { + /// Construct a retryable application failure with no extra metadata. + pub fn new(source: impl Into) -> Self { + Self { + source: source.into(), + type_name: None, + non_retryable: false, + next_retry_delay: None, + category: ApplicationErrorCategory::Unspecified, + details: None, + failure: None, + cause: None, + } + } + + /// Construct a non-retryable application failure with no extra metadata. + pub fn non_retryable(source: impl Into) -> Self { + Self { + non_retryable: true, + ..Self::new(source) + } + } + + /// Returns the wrapped source error. + pub fn source_error(&self) -> &anyhow::Error { + &self.source + } + + /// Returns the configured application failure type name, if any. + pub fn type_name(&self) -> Option<&str> { + self.type_name.as_deref() + } + + /// Returns true if this failure should be treated as non-retryable. + pub fn is_non_retryable(&self) -> bool { + self.non_retryable + } + + /// Returns the explicitly configured next retry delay, if any. + pub fn next_retry_delay(&self) -> Option { + self.next_retry_delay + } + + /// Returns the application error category. + pub fn category(&self) -> ApplicationErrorCategory { + self.category + } + + /// Returns the decoded details deserialized as the requested type, if any. + pub fn details( + &self, + ) -> Result, PayloadConversionError> { + self.details + .as_ref() + .map(FailurePayloads::deserialize) + .transpose() + } + + /// Returns the raw decoded details payloads, if any. + pub fn raw_details(&self) -> Option<&[Payload]> { + self.details.as_ref().and_then(FailurePayloads::raw) + } + + pub(crate) fn failure_payloads(&self) -> Option<&FailurePayloads> { + self.details.as_ref() + } + + /// Returns the original failure proto when this application failure was decoded from one. + pub fn failure(&self) -> Option<&Failure> { + self.failure.as_ref() + } + + /// Consumes this application failure and returns the retained proto failure, if one exists. + pub fn into_failure(self) -> Option { + self.failure + } + + /// Returns the normalized cause, if any. + pub fn cause(&self) -> Option<&IncomingError> { + self.cause.as_deref() + } + + /// If this [`ApplicationFailure`] was caused by a timeout, returns the associated + /// [`TimeoutError`]. + pub fn as_timeout(&self) -> Option<&TimeoutError> { + self.cause().and_then(IncomingError::as_timeout) + } + + /// If this [`ApplicationFailure`] was caused by a cancellation, returns the associated + /// [`CancelledError`]. + pub fn as_cancelled(&self) -> Option<&CancelledError> { + self.cause().and_then(IncomingError::as_cancelled) + } + + pub(crate) fn from_failure( + failure: Failure, + cause: Option, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Self { + let app_info = failure + .maybe_application_failure() + .cloned() + .unwrap_or_default(); + let type_name = (!app_info.r#type.is_empty()).then_some(app_info.r#type.clone()); + Self { + source: anyhow::anyhow!(failure.message.clone()), + type_name, + non_retryable: app_info.non_retryable, + next_retry_delay: app_info.next_retry_delay.and_then(|d| d.try_into().ok()), + category: app_info.category(), + details: app_info.details.map(|details| { + FailurePayloads::from(DecodablePayloads::new( + details.payloads, + payload_converter.clone(), + *context, + )) + }), + failure: Some(failure), + cause: cause.map(Box::new), + } + } +} + +impl std::fmt::Display for ApplicationFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.source) + } +} + +impl std::error::Error for ApplicationFailure { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.cause + .as_deref() + .map(|cause| cause as &(dyn std::error::Error + 'static)) + .or_else(|| Some(self.source.as_ref())) + } +} + +impl From for ApplicationFailure { + fn from(value: anyhow::Error) -> Self { + Self::new(value) + } +} + +impl From for ApplicationFailure { + fn from(value: PayloadConversionError) -> Self { + Self::new(value) + } +} + +/// A typed outbound error surface used before encoding to a Temporal failure proto. +#[derive(Debug, thiserror::Error)] +pub enum OutgoingError { + /// An error produced while completing an activity. + #[error(transparent)] + Activity(#[from] OutgoingActivityError), + /// An error produced from a workflow. + #[error(transparent)] + Workflow(#[from] OutgoingWorkflowError), +} + +/// A typed outbound activity error. +#[derive(Debug, thiserror::Error)] +pub enum OutgoingActivityError { + /// An activity application failure. + #[error(transparent)] + Application(#[from] Box), + /// An activity cancellation with optional details. + #[error("Activity cancelled")] + Cancelled { + /// Optional cancellation details. + details: Option, + }, +} + +/// A typed outbound workflow failure. +#[derive(Debug, thiserror::Error)] +pub enum OutgoingWorkflowError { + /// A workflow application failure. + #[error(transparent)] + Application(#[from] Box), + /// A workflow failure sourced from an activity execution. + #[error(transparent)] + ActivityExecution(#[from] Box), + /// A workflow failure sourced from a child-workflow execution. + #[error(transparent)] + ChildWorkflowExecution(#[from] Box), + /// A workflow failure sourced from child-workflow start. + #[error(transparent)] + ChildWorkflowStart(#[from] Box), + /// A workflow failure sourced from child-workflow signaling. + #[error(transparent)] + ChildWorkflowSignal(#[from] Box), +} + +impl From for OutgoingWorkflowError { + fn from(value: anyhow::Error) -> Self { + Self::Application(Box::new(ApplicationFailure::new(value))) + } +} + +impl From for OutgoingWorkflowError { + fn from(value: PayloadConversionError) -> Self { + Self::Application(Box::new(value.into())) + } +} + +impl From for OutgoingWorkflowError { + fn from(value: ApplicationFailure) -> Self { + Self::Application(Box::new(value)) + } +} + +impl From for OutgoingWorkflowError { + fn from(value: ActivityExecutionError) -> Self { + Self::ActivityExecution(Box::new(value)) + } +} + +impl From for OutgoingWorkflowError { + fn from(value: ChildWorkflowExecutionError) -> Self { + Self::ChildWorkflowExecution(Box::new(value)) + } +} + +impl From for OutgoingWorkflowError { + fn from(value: ChildWorkflowStartError) -> Self { + Self::ChildWorkflowStart(Box::new(value)) + } +} + +impl From for OutgoingWorkflowError { + fn from(value: ChildWorkflowSignalError) -> Self { + Self::ChildWorkflowSignal(Box::new(value)) + } +} + +/// A normalized incoming Temporal failure decoded from a protobuf [`Failure`]. +#[derive(Debug)] +pub enum IncomingError { + /// A decoded application failure. + Application(ApplicationFailure), + /// A decoded timeout failure. + Timeout(TimeoutError), + /// A decoded cancellation failure. + Cancelled(CancelledError), + /// A decoded terminated failure. + Terminated(TerminatedError), + /// A decoded server failure. + Server(ServerError), + /// A decoded reset-workflow failure. + ResetWorkflow(ResetWorkflowError), + /// A decoded activity failure wrapper. + Activity(ActivityFailureError), + /// A decoded child-workflow failure wrapper. + ChildWorkflowExecution(ChildWorkflowFailureError), + /// A decoded nexus operation failure wrapper. + NexusOperationExecution(IncomingNexusOperationExecutionError), + /// A decoded nexus handler failure wrapper. + NexusHandler(IncomingNexusHandlerError), +} + +impl IncomingError { + /// Returns the original failure proto for this normalized error. + pub fn failure(&self) -> &Failure { + match self { + IncomingError::Application(err) => err + .failure() + .expect("decoded application failures retain their original proto"), + IncomingError::Timeout(err) => err.failure(), + IncomingError::Cancelled(err) => err.failure(), + IncomingError::Terminated(err) => err.failure(), + IncomingError::Server(err) => err.failure(), + IncomingError::ResetWorkflow(err) => err.failure(), + IncomingError::Activity(err) => err.failure(), + IncomingError::ChildWorkflowExecution(err) => err.failure(), + IncomingError::NexusOperationExecution(err) => err.failure(), + IncomingError::NexusHandler(err) => err.failure(), + } + } + + /// Returns the normalized cause, if any. + pub fn cause(&self) -> Option<&IncomingError> { + match self { + IncomingError::Application(err) => err.cause(), + IncomingError::Timeout(err) => err.cause(), + IncomingError::Cancelled(err) => err.cause(), + IncomingError::Terminated(err) => err.cause(), + IncomingError::Server(err) => err.cause(), + IncomingError::ResetWorkflow(err) => err.cause(), + IncomingError::Activity(err) => err.cause(), + IncomingError::ChildWorkflowExecution(err) => err.cause(), + IncomingError::NexusOperationExecution(err) => err.cause(), + IncomingError::NexusHandler(err) => err.cause(), + } + } + + /// Consumes this normalized error and returns the retained proto failure. + pub fn into_failure(self) -> Failure { + match self { + IncomingError::Application(err) => err + .into_failure() + .expect("decoded application failures retain their original proto"), + IncomingError::Timeout(err) => err.into_failure(), + IncomingError::Cancelled(err) => err.into_failure(), + IncomingError::Terminated(err) => err.into_failure(), + IncomingError::Server(err) => err.into_failure(), + IncomingError::ResetWorkflow(err) => err.into_failure(), + IncomingError::Activity(err) => err.into_failure(), + IncomingError::ChildWorkflowExecution(err) => err.into_failure(), + IncomingError::NexusOperationExecution(err) => err.into_failure(), + IncomingError::NexusHandler(err) => err.into_failure(), + } + } + + /// If the [`IncomingError`] is a timeout, returns the associated [`TimeoutError`]. + pub fn as_timeout(&self) -> Option<&TimeoutError> { + match self { + IncomingError::Timeout(err) => Some(err), + _ => None, + } + } + + /// If the [`IncomingError`] is a cancellation, returns the associated [`CancelledError`]. + pub fn as_cancelled(&self) -> Option<&CancelledError> { + match self { + IncomingError::Cancelled(err) => Some(err), + _ => None, + } + } +} + +impl std::fmt::Display for IncomingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + IncomingError::Application(err) => err.fmt(f), + IncomingError::Timeout(err) => err.fmt(f), + IncomingError::Cancelled(err) => err.fmt(f), + IncomingError::Terminated(err) => err.fmt(f), + IncomingError::Server(err) => err.fmt(f), + IncomingError::ResetWorkflow(err) => err.fmt(f), + IncomingError::Activity(err) => err.fmt(f), + IncomingError::ChildWorkflowExecution(err) => err.fmt(f), + IncomingError::NexusOperationExecution(err) => err.fmt(f), + IncomingError::NexusHandler(err) => err.fmt(f), + } + } +} + +impl std::error::Error for IncomingError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + IncomingError::Application(err) => Some(err), + IncomingError::Timeout(err) => Some(err), + IncomingError::Cancelled(err) => Some(err), + IncomingError::Terminated(err) => Some(err), + IncomingError::Server(err) => Some(err), + IncomingError::ResetWorkflow(err) => Some(err), + IncomingError::Activity(err) => Some(err), + IncomingError::ChildWorkflowExecution(err) => Some(err), + IncomingError::NexusOperationExecution(err) => Some(err), + IncomingError::NexusHandler(err) => Some(err), + } + } +} + +macro_rules! impl_incoming_failure_wrapper { + ($name:ident) => { + impl $name { + /// Returns the original failure proto. + pub fn failure(&self) -> &Failure { + &self.failure + } + + /// Returns the normalized cause, if any. + pub fn cause(&self) -> Option<&IncomingError> { + self.cause.as_deref() + } + + /// Consumes this wrapper and returns the retained proto failure. + pub fn into_failure(self) -> Failure { + self.failure + } + + /// Consumes this wrapper and returns the retained proto failure and normalized cause. + pub fn into_parts(self) -> (Failure, Option) { + (self.failure, self.cause.map(|cause| *cause)) + } + } + + impl std::fmt::Display for $name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.failure.fmt(f) + } + } + + impl std::error::Error for $name { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.cause + .as_deref() + .map(|cause| cause as &(dyn std::error::Error + 'static)) + } + } + }; +} + +macro_rules! incoming_failure_wrapper { + ($name:ident, $doc:literal) => { + #[doc = $doc] + #[derive(Debug)] + pub struct $name { + failure: Failure, + cause: Option>, + } + + impl $name { + /// Creates a new normalized incoming error wrapper. + pub(crate) fn new(failure: Failure, cause: Option) -> Self { + Self { + failure, + cause: cause.map(Box::new), + } + } + } + + impl_incoming_failure_wrapper!($name); + }; +} + +/// A normalized timeout failure. +#[derive(Debug)] +pub struct TimeoutError { + failure: Failure, + cause: Option>, + timeout_type: TimeoutType, + last_heartbeat_details: Option, +} + +impl TimeoutError { + /// Creates a new normalized timeout error wrapper. + pub(crate) fn new( + failure: Failure, + failure_info: crate::protos::temporal::api::failure::v1::TimeoutFailureInfo, + cause: Option, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Self { + Self { + failure, + cause: cause.map(Box::new), + timeout_type: failure_info.timeout_type(), + last_heartbeat_details: failure_info.last_heartbeat_details.map(|details| { + DecodablePayloads::new(details.payloads, payload_converter.clone(), *context) + }), + } + } + + /// Returns the timeout kind described by the failure. + pub fn timeout_type(&self) -> TimeoutType { + self.timeout_type + } + + /// Returns the last heartbeat details carried by the timeout, if any. + pub fn last_heartbeat_details( + &self, + ) -> Result, PayloadConversionError> { + self.last_heartbeat_details + .as_ref() + .map(DecodablePayloads::deserialize) + .transpose() + } + + /// Returns the raw decoded heartbeat details carried by the timeout, if any. + pub fn raw_last_heartbeat_details(&self) -> Option<&[Payload]> { + self.last_heartbeat_details + .as_ref() + .map(DecodablePayloads::raw) + } +} + +impl_incoming_failure_wrapper!(TimeoutError); + +/// A normalized cancellation failure. +#[derive(Debug)] +pub struct CancelledError { + failure: Failure, + cause: Option>, + details: Option, +} + +impl CancelledError { + /// Creates a new normalized cancellation error wrapper. + pub(crate) fn new( + failure: Failure, + failure_info: crate::protos::temporal::api::failure::v1::CanceledFailureInfo, + cause: Option, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Self { + Self { + failure, + cause: cause.map(Box::new), + details: failure_info.details.map(|details| { + DecodablePayloads::new(details.payloads, payload_converter.clone(), *context) + }), + } + } + + /// Returns the cancellation details carried by the failure, deserialized as the requested + /// type, if any. + pub fn details( + &self, + ) -> Result, PayloadConversionError> { + self.details + .as_ref() + .map(DecodablePayloads::deserialize) + .transpose() + } + + /// Returns the raw decoded cancellation details carried by the failure, if any. + pub fn raw_details(&self) -> Option<&[Payload]> { + self.details.as_ref().map(DecodablePayloads::raw) + } +} + +impl_incoming_failure_wrapper!(CancelledError); +incoming_failure_wrapper!(TerminatedError, "A normalized terminated failure."); +incoming_failure_wrapper!(ServerError, "A normalized server failure."); +incoming_failure_wrapper!(ResetWorkflowError, "A normalized reset-workflow failure."); + +/// A normalized activity failure wrapper. +#[derive(Debug)] +pub struct ActivityFailureError { + failure: Failure, + cause: Option>, + activity_id: String, + activity_type: Option, + scheduled_event_id: i64, + started_event_id: i64, + identity: String, + retry_state: crate::protos::temporal::api::enums::v1::RetryState, +} + +impl ActivityFailureError { + /// Creates a new normalized activity failure wrapper. + pub(crate) fn new( + failure: Failure, + failure_info: crate::protos::temporal::api::failure::v1::ActivityFailureInfo, + cause: Option, + ) -> Self { + let retry_state = failure_info.retry_state(); + Self { + failure, + cause: cause.map(Box::new), + activity_id: failure_info.activity_id, + activity_type: failure_info.activity_type, + scheduled_event_id: failure_info.scheduled_event_id, + started_event_id: failure_info.started_event_id, + identity: failure_info.identity, + retry_state, + } + } + + /// Returns the activity id reported by the failure. + pub fn activity_id(&self) -> &str { + &self.activity_id + } + + /// Returns the activity type, if present. + pub fn activity_type(&self) -> Option<&crate::protos::temporal::api::common::v1::ActivityType> { + self.activity_type.as_ref() + } + + /// Returns the scheduled event id. + pub fn scheduled_event_id(&self) -> i64 { + self.scheduled_event_id + } + + /// Returns the started event id. + pub fn started_event_id(&self) -> i64 { + self.started_event_id + } + + /// Returns the worker identity captured on the failure. + pub fn identity(&self) -> &str { + &self.identity + } + + /// Returns the retry state reported by core. + pub fn retry_state(&self) -> crate::protos::temporal::api::enums::v1::RetryState { + self.retry_state + } + + /// If this [`ActivityFailureError`] was caused by a timeout, returns the associated + /// [`TimeoutError`]. + pub fn as_timeout(&self) -> Option<&TimeoutError> { + self.cause().and_then(IncomingError::as_timeout) + } + + /// If this [`ActivityFailureError`] was caused by a cancellation, returns the associated + /// [`CancelledError`]. + pub fn as_cancelled(&self) -> Option<&CancelledError> { + self.cause().and_then(IncomingError::as_cancelled) + } +} + +impl_incoming_failure_wrapper!(ActivityFailureError); +/// A normalized child-workflow execution failure wrapper. +#[derive(Debug)] +pub struct ChildWorkflowFailureError { + failure: Failure, + cause: Option>, + namespace: String, + workflow_execution: Option, + workflow_type: Option, + initiated_event_id: i64, + started_event_id: i64, + retry_state: crate::protos::temporal::api::enums::v1::RetryState, +} + +impl ChildWorkflowFailureError { + /// Creates a new normalized child-workflow execution failure wrapper. + pub(crate) fn new( + failure: Failure, + failure_info: crate::protos::temporal::api::failure::v1::ChildWorkflowExecutionFailureInfo, + cause: Option, + ) -> Self { + let retry_state = failure_info.retry_state(); + Self { + failure, + cause: cause.map(Box::new), + namespace: failure_info.namespace, + workflow_execution: failure_info.workflow_execution, + workflow_type: failure_info.workflow_type, + initiated_event_id: failure_info.initiated_event_id, + started_event_id: failure_info.started_event_id, + retry_state, + } + } + + /// Returns the namespace of the child workflow. + pub fn namespace(&self) -> &str { + &self.namespace + } + + /// Returns the child workflow execution, if present. + pub fn workflow_execution( + &self, + ) -> Option<&crate::protos::temporal::api::common::v1::WorkflowExecution> { + self.workflow_execution.as_ref() + } + + /// Returns the child workflow type, if present. + pub fn workflow_type(&self) -> Option<&crate::protos::temporal::api::common::v1::WorkflowType> { + self.workflow_type.as_ref() + } + + /// Returns the initiated event id. + pub fn initiated_event_id(&self) -> i64 { + self.initiated_event_id + } + + /// Returns the started event id. + pub fn started_event_id(&self) -> i64 { + self.started_event_id + } + + /// Returns the retry state reported by core. + pub fn retry_state(&self) -> crate::protos::temporal::api::enums::v1::RetryState { + self.retry_state + } + + /// If this [`ChildWorkflowFailureError`] was caused by a timeout, returns the associated + /// [`TimeoutError`]. + pub fn as_timeout(&self) -> Option<&TimeoutError> { + self.cause().and_then(IncomingError::as_timeout) + } + + /// If this [`ChildWorkflowFailureError`] was caused by a cancellation, returns the associated + /// [`CancelledError`]. + pub fn as_cancelled(&self) -> Option<&CancelledError> { + self.cause().and_then(IncomingError::as_cancelled) + } +} + +impl_incoming_failure_wrapper!(ChildWorkflowFailureError); +incoming_failure_wrapper!( + IncomingNexusOperationExecutionError, + "A normalized nexus operation failure wrapper." +); +incoming_failure_wrapper!( + IncomingNexusHandlerError, + "A normalized nexus handler failure wrapper." +); + +/// Error type for activity execution outcomes. +#[derive(Debug, thiserror::Error)] +pub enum ActivityExecutionError { + /// The activity failed with the given failure details. + #[error("Activity failed: {}", .0.failure().message)] + Failed(#[source] ActivityFailureError), + /// The activity was cancelled. + #[error("Activity cancelled: {}", .0.failure().message)] + Cancelled(#[source] CancelledError), + /// Failed to serialize input or deserialize result payload. + #[error("Payload conversion failed: {0}")] + Serialization(#[from] PayloadConversionError), +} + +impl ActivityExecutionError { + /// Returns the retained top-level activity failure proto, if one exists. + pub fn failure(&self) -> Option<&Failure> { + match self { + ActivityExecutionError::Failed(err) => Some(err.failure()), + ActivityExecutionError::Cancelled(err) => Some(err.failure()), + ActivityExecutionError::Serialization(_) => None, + } + } + + /// Returns the normalized cause of the top-level activity failure, if any. + pub fn cause(&self) -> Option<&IncomingError> { + match self { + ActivityExecutionError::Failed(err) => err.cause(), + ActivityExecutionError::Cancelled(err) => err.cause(), + ActivityExecutionError::Serialization(_) => None, + } + } + + /// Returns the underlying failure reason for wrapper-shaped activity failures. + pub fn reason(&self) -> Option<&IncomingError> { + match self { + ActivityExecutionError::Failed(err) => err.cause(), + ActivityExecutionError::Cancelled(_) | ActivityExecutionError::Serialization(_) => None, + } + } + + /// If this [`ActivityExecutionError`] was caused by a timeout, returns the associated + /// [`TimeoutError`]. + pub fn as_timeout(&self) -> Option<&TimeoutError> { + match self { + ActivityExecutionError::Failed(err) => err.as_timeout(), + ActivityExecutionError::Serialization(_) | ActivityExecutionError::Cancelled(_) => None, + } + } + + /// If this [`ActivityExecutionError`] was caused by a cancellation, returns the associated + /// [`CancelledError`]. + pub fn as_cancelled(&self) -> Option<&CancelledError> { + match self { + ActivityExecutionError::Failed(err) => err.as_cancelled(), + ActivityExecutionError::Cancelled(err) => Some(err), + ActivityExecutionError::Serialization(_) => None, + } + } +} + +/// Error returned when starting a child workflow fails. +#[derive(Debug, thiserror::Error)] +pub enum ChildWorkflowStartError { + /// The child workflow start was cancelled before the normal execution wrapper path existed. + #[error("Child workflow start cancelled: {}", .0.failure().message)] + Cancelled(#[source] Box), + /// The child workflow failed to start (e.g., workflow ID already exists). + #[error( + "Child workflow start failed: workflow_id={workflow_id}, workflow_type={workflow_type}, cause={cause:?}" + )] + StartFailed { + /// The workflow ID that was requested. + workflow_id: String, + /// The workflow type that was requested. + workflow_type: String, + /// The cause of the start failure. + cause: StartChildWorkflowExecutionFailedCause, + }, + /// Failed to serialize child workflow input payloads. + #[error("Payload conversion failed: {0}")] + Serialization(#[from] PayloadConversionError), +} + +impl ChildWorkflowStartError { + /// Returns the retained top-level failure proto, if one exists. + pub fn failure(&self) -> Option<&Failure> { + match self { + ChildWorkflowStartError::Cancelled(err) => Some(err.failure()), + ChildWorkflowStartError::StartFailed { .. } + | ChildWorkflowStartError::Serialization(_) => None, + } + } + + /// Returns the normalized cause of the retained failure proto, if any. + pub fn cause(&self) -> Option<&IncomingError> { + match self { + ChildWorkflowStartError::Cancelled(err) => err.cause(), + ChildWorkflowStartError::StartFailed { .. } + | ChildWorkflowStartError::Serialization(_) => None, + } + } +} + +/// Error returned when a child workflow execution fails. +#[derive(Debug, thiserror::Error)] +pub enum ChildWorkflowExecutionError { + /// The child workflow failed. + #[error("Child workflow failed: {}", .0.failure().message)] + Failed(#[source] Box), + /// Failed to serialize input or deserialize the child workflow result payload. + #[error("Payload conversion failed: {0}")] + Serialization(#[from] PayloadConversionError), +} + +impl ChildWorkflowExecutionError { + /// Returns the retained top-level child-workflow failure proto, if one exists. + pub fn failure(&self) -> Option<&Failure> { + match self { + ChildWorkflowExecutionError::Failed(err) => Some(err.failure()), + ChildWorkflowExecutionError::Serialization(_) => None, + } + } + + /// Returns the normalized cause of the top-level child-workflow failure, if any. + pub fn cause(&self) -> Option<&IncomingError> { + match self { + ChildWorkflowExecutionError::Failed(err) => err.cause(), + ChildWorkflowExecutionError::Serialization(_) => None, + } + } + + /// Returns the underlying failure reason for wrapper-shaped child-workflow failures. + pub fn reason(&self) -> Option<&IncomingError> { + match self { + ChildWorkflowExecutionError::Failed(err) => err.cause(), + ChildWorkflowExecutionError::Serialization(_) => None, + } + } + + /// If this [`ChildWorkflowExecutionError`] was caused by a timeout, returns the associated + /// [`TimeoutError`]. + pub fn as_timeout(&self) -> Option<&TimeoutError> { + match self { + ChildWorkflowExecutionError::Failed(err) => err.as_timeout(), + ChildWorkflowExecutionError::Serialization(_) => None, + } + } + + /// If this [`ChildWorkflowExecutionError`] was caused by a cancellation, returns the associated + /// [`CancelledError`]. + pub fn as_cancelled(&self) -> Option<&CancelledError> { + match self { + ChildWorkflowExecutionError::Failed(err) => err.as_cancelled(), + ChildWorkflowExecutionError::Serialization(_) => None, + } + } +} + +/// Error returned when signaling a child workflow fails. +#[derive(Debug, thiserror::Error)] +pub enum ChildWorkflowSignalError { + /// The signal delivery failed. + #[error("Child workflow signal failed: {}", .0.failure().message)] + Failed(#[source] Box), + /// Failed to serialize the signal input payload. + #[error("Signal payload conversion failed: {0}")] + Serialization(#[from] PayloadConversionError), +} + +impl ChildWorkflowSignalError { + /// Returns the retained top-level child-workflow signal failure proto, if one exists. + pub fn failure(&self) -> Option<&Failure> { + match self { + ChildWorkflowSignalError::Failed(err) => Some(err.failure()), + ChildWorkflowSignalError::Serialization(_) => None, + } + } + + /// Returns the normalized cause of the child-workflow signal failure, if any. + pub fn cause(&self) -> Option<&IncomingError> { + match self { + ChildWorkflowSignalError::Failed(err) => err.cause(), + ChildWorkflowSignalError::Serialization(_) => None, + } + } + + /// Returns the underlying failure reason for wrapper-shaped signal failures. + pub fn reason(&self) -> Option<&IncomingError> { + match self { + ChildWorkflowSignalError::Failed(err) => Some(err.error()), + ChildWorkflowSignalError::Serialization(_) => None, + } + } +} + +/// A normalized child-workflow signal failure wrapper. +#[derive(Debug)] +pub struct ChildWorkflowSignalFailureError { + failure: Failure, + error: Box, +} + +impl ChildWorkflowSignalFailureError { + /// Creates a child-workflow signal failure wrapper. + pub(crate) fn new(failure: Failure, error: IncomingError) -> Self { + Self { + failure, + error: Box::new(error), + } + } + + /// Returns the retained top-level proto failure. + pub fn failure(&self) -> &Failure { + &self.failure + } + + /// Returns the normalized direct cause of the child-workflow signal failure, if any. + pub fn cause(&self) -> Option<&IncomingError> { + self.error.cause() + } + + /// Returns the direct decoded incoming error represented by the top-level proto failure. + pub fn error(&self) -> &IncomingError { + &self.error + } +} + +impl std::fmt::Display for ChildWorkflowSignalFailureError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.failure.fmt(f) + } +} + +impl std::error::Error for ChildWorkflowSignalFailureError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.cause() + .map(|cause| cause as &(dyn std::error::Error + 'static)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + data_converters::{ + DefaultFailureConverter, FailureConverter, GenericPayloadConverter, PayloadConverter, + SerializationContext, SerializationContextData, + }, + protos::temporal::api::{common::v1::Payload, failure::v1::failure::FailureInfo}, + }; + + struct AlwaysFailsSerialize; + + impl serde::Serialize for AlwaysFailsSerialize { + fn serialize(&self, _serializer: S) -> Result { + Err(serde::ser::Error::custom("serialize boom")) + } + } + + #[test] + fn constructors_set_retryability_defaults() { + assert!(!ApplicationFailure::new(anyhow::anyhow!("retryable")).is_non_retryable()); + assert!( + ApplicationFailure::non_retryable(anyhow::anyhow!("non-retryable")).is_non_retryable() + ); + } + + #[test] + fn conversion_preserves_application_metadata() { + let payloads = Payloads { + payloads: vec![Payload { + data: b"details".to_vec(), + ..Default::default() + }], + }; + let failure = DefaultFailureConverter.to_failure( + OutgoingError::Workflow(OutgoingWorkflowError::Application(Box::new( + ApplicationFailure::builder(anyhow::anyhow!("oops")) + .type_name("MyType".to_owned()) + .non_retryable(true) + .next_retry_delay(Duration::from_secs(3)) + .category(ApplicationErrorCategory::Benign) + .details(RawValue::new(payloads.payloads.clone())) + .build(), + ))), + &PayloadConverter::default(), + &SerializationContextData::None, + ); + let Some(FailureInfo::ApplicationFailureInfo(info)) = failure.failure_info else { + panic!("expected application failure info"); + }; + assert_eq!(failure.message, "oops"); + assert_eq!(info.r#type, "MyType"); + assert!(info.non_retryable); + assert_eq!(info.details, Some(payloads)); + assert_eq!(info.category(), ApplicationErrorCategory::Benign); + assert_eq!(info.next_retry_delay.unwrap().seconds, 3); + } + + #[test] + fn builder_accepts_raw_payload_details() { + let payload = Payload { + data: b"details".to_vec(), + ..Default::default() + }; + let failure = DefaultFailureConverter.to_failure( + OutgoingError::Workflow(OutgoingWorkflowError::Application(Box::new( + ApplicationFailure::builder(anyhow::anyhow!("oops")) + .details(RawValue::new(vec![payload.clone()])) + .build(), + ))), + &PayloadConverter::default(), + &SerializationContextData::None, + ); + + let Some(FailureInfo::ApplicationFailureInfo(info)) = failure.failure_info else { + panic!("expected application failure info"); + }; + assert_eq!(info.details.unwrap().payloads, vec![payload]); + } + + #[test] + fn builder_accepts_serializable_details() { + let failure = DefaultFailureConverter.to_failure( + OutgoingError::Workflow(OutgoingWorkflowError::Application(Box::new( + ApplicationFailure::builder(anyhow::anyhow!("oops")) + .details("details".to_string()) + .build(), + ))), + &PayloadConverter::default(), + &SerializationContextData::None, + ); + + let Some(FailureInfo::ApplicationFailureInfo(info)) = failure.failure_info else { + panic!("expected application failure info"); + }; + let payloads = info.details.expect("expected details").payloads; + let converter = PayloadConverter::default(); + let details: String = converter + .from_payloads( + &SerializationContext { + data: &SerializationContextData::None, + converter: &converter, + }, + payloads, + ) + .unwrap(); + assert_eq!(details, "details"); + } + + #[test] + fn application_failure_encoding_surfaces_detail_encoding_errors() { + let failure = DefaultFailureConverter.to_failure( + OutgoingError::Workflow(OutgoingWorkflowError::Application(Box::new( + ApplicationFailure::builder(anyhow::anyhow!("oops")) + .details(AlwaysFailsSerialize) + .build(), + ))), + &PayloadConverter::default(), + &SerializationContextData::None, + ); + + assert_eq!( + failure.message, + "Failed converting error to failure: Encoding error: serialize boom, original error message: oops" + ); + assert!(matches!( + failure.failure_info, + Some(FailureInfo::ApplicationFailureInfo(_)) + )); + } + + #[test] + fn anyhow_workflow_errors_default_to_application_outgoing_errors() { + let outgoing: OutgoingWorkflowError = anyhow::anyhow!("workflow boom").into(); + + let OutgoingWorkflowError::Application(app) = outgoing else { + panic!("plain workflow errors should default to application failures"); + }; + assert_eq!(app.to_string(), "workflow boom"); + } + + #[test] + fn payload_conversion_errors_default_to_application_outgoing_errors() { + let outgoing: OutgoingWorkflowError = + PayloadConversionError::EncodingError(anyhow::anyhow!("encode boom").into()).into(); + + let OutgoingWorkflowError::Application(app) = outgoing else { + panic!("payload conversion errors should default to application failures"); + }; + assert_eq!(app.to_string(), "Encoding error: encode boom"); + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index c9127778f..ffdc299fd 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -11,6 +11,7 @@ mod activity_definition; pub mod data_converters; #[cfg(feature = "envconfig")] pub mod envconfig; +pub mod error; #[doc(hidden)] pub mod fsm_trait; pub mod payload_visitor; diff --git a/crates/common/src/payload_visitor.rs b/crates/common/src/payload_visitor.rs index dc0d2e78b..209502b9a 100644 --- a/crates/common/src/payload_visitor.rs +++ b/crates/common/src/payload_visitor.rs @@ -193,25 +193,32 @@ include!(concat!(env!("OUT_DIR"), "/payload_visitor_impl.rs")); #[cfg(test)] mod tests { use super::*; - use crate::protos::{ - coresdk::{ - activity_result::{ - ActivityResolution, Success, activity_resolution::Status as ActivityStatus, - }, - workflow_activation::{ - InitializeWorkflow, ResolveActivity, WorkflowActivation, WorkflowActivationJob, - workflow_activation_job::Variant, - }, - workflow_commands::{ - ContinueAsNewWorkflowExecution, ScheduleActivity, StartChildWorkflowExecution, - UpsertWorkflowSearchAttributes, WorkflowCommand, - workflow_command::Variant as CmdVariant, + use crate::{ + data_converters::{DefaultFailureConverter, FailureConverter, PayloadConverter}, + error::{ApplicationFailure, OutgoingError, OutgoingWorkflowError}, + protos::{ + coresdk::{ + activity_result::{ + ActivityResolution, Success, activity_resolution::Status as ActivityStatus, + }, + workflow_activation::{ + InitializeWorkflow, ResolveActivity, WorkflowActivation, WorkflowActivationJob, + workflow_activation_job::Variant, + }, + workflow_commands::{ + ContinueAsNewWorkflowExecution, ScheduleActivity, StartChildWorkflowExecution, + UpsertWorkflowSearchAttributes, WorkflowCommand, + workflow_command::Variant as CmdVariant, + }, + workflow_completion::{ + WorkflowActivationCompletion, workflow_activation_completion::Status, + }, }, - workflow_completion::{ - WorkflowActivationCompletion, workflow_activation_completion::Status, + temporal::api::{ + common::v1::{Memo, SearchAttributes}, + failure::v1::failure::FailureInfo, }, }, - temporal::api::common::v1::{Memo, SearchAttributes}, }; use futures::FutureExt; use std::collections::HashMap; @@ -645,4 +652,31 @@ mod tests { assert!(is_encoded(p), "payload {} should be encoded", i); } } + + #[tokio::test] + async fn test_encode_failure_encodes_application_failure_details() { + let mut failure = DefaultFailureConverter.to_failure( + OutgoingError::Workflow(OutgoingWorkflowError::Application(Box::new( + ApplicationFailure::builder(anyhow::anyhow!("app boom")) + .details(crate::data_converters::RawValue::new(vec![make_payload( + "detail", + )])) + .build(), + ))), + &PayloadConverter::default(), + &SerializationContextData::Workflow, + ); + + encode_payloads( + &mut failure, + &MarkingCodec, + &SerializationContextData::Workflow, + ) + .await; + + let Some(FailureInfo::ApplicationFailureInfo(info)) = failure.failure_info else { + panic!("expected application failure info") + }; + assert!(is_encoded(&info.details.unwrap().payloads[0])); + } } diff --git a/crates/common/src/protos/mod.rs b/crates/common/src/protos/mod.rs index b07784308..6e39904f8 100644 --- a/crates/common/src/protos/mod.rs +++ b/crates/common/src/protos/mod.rs @@ -167,6 +167,14 @@ pub mod coresdk { } } + pub fn cancel(fail: APIFailure) -> Self { + Self { + status: Some(aer::Status::Cancelled(Cancellation { + failure: Some(fail), + })), + } + } + pub const fn will_complete_async() -> Self { Self { status: Some(aer::Status::WillCompleteAsync(WillCompleteAsync {})), diff --git a/crates/macros/src/workflow_definitions.rs b/crates/macros/src/workflow_definitions.rs index 75ccf96ce..b37a47b08 100644 --- a/crates/macros/src/workflow_definitions.rs +++ b/crates/macros/src/workflow_definitions.rs @@ -1012,7 +1012,7 @@ impl WorkflowMethodsDefinition { let result = #run_call; match result { Ok(value) => ::temporalio_sdk::workflows::serialize_result(value, &ctx.payload_converter()) - .map_err(|e| ::temporalio_sdk::WorkflowTermination::failed(e)), + .map_err(|e| ::temporalio_sdk::WorkflowTermination::from(::anyhow::Error::new(e))), Err(e) => Err(e), } }.boxed_local() diff --git a/crates/sdk-core/src/core_tests/workers.rs b/crates/sdk-core/src/core_tests/workers.rs index 6f8f6436f..db6ec59a5 100644 --- a/crates/sdk-core/src/core_tests/workers.rs +++ b/crates/sdk-core/src/core_tests/workers.rs @@ -15,14 +15,14 @@ use crate::{ }, }; use futures_util::{stream, stream::StreamExt}; -use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, -}; -use std::{cell::RefCell, collections::HashMap, time::Duration}; -use temporalio_common::protos::temporal::api::{ - namespace::v1::{NamespaceInfo, namespace_info::Capabilities}, - workflowservice::v1::DescribeNamespaceResponse, +use std::{ + cell::RefCell, + collections::HashMap, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, }; use temporalio_common::{ protos::{ @@ -41,6 +41,7 @@ use temporalio_common::{ ApplicationFailureInfo, CanceledFailureInfo, Failure, NexusHandlerFailureInfo, TimeoutFailureInfo, failure::FailureInfo, }, + namespace::v1::{NamespaceInfo, namespace_info::Capabilities}, nexus::{ self, v1::{ @@ -50,17 +51,17 @@ use temporalio_common::{ }, }, workflowservice::v1::{ - PollActivityTaskQueueResponse, PollNexusTaskQueueResponse, - PollWorkflowTaskQueueResponse, RespondActivityTaskCompletedResponse, - RespondNexusTaskCompletedResponse, RespondNexusTaskFailedResponse, - RespondWorkflowTaskCompletedResponse, ShutdownWorkerResponse, + DescribeNamespaceResponse, PollActivityTaskQueueResponse, + PollNexusTaskQueueResponse, PollWorkflowTaskQueueResponse, + RespondActivityTaskCompletedResponse, RespondNexusTaskCompletedResponse, + RespondNexusTaskFailedResponse, RespondWorkflowTaskCompletedResponse, + ShutdownWorkerResponse, }, }, }, worker::WorkerTaskTypes, }; -use tokio::sync::Notify; -use tokio::sync::{Barrier, watch}; +use tokio::sync::{Barrier, Notify, watch}; use uuid::Uuid; #[tokio::test] diff --git a/crates/sdk-core/tests/integ_tests/async_activity_client_tests.rs b/crates/sdk-core/tests/integ_tests/async_activity_client_tests.rs index 6f95ba75b..09ec810e6 100644 --- a/crates/sdk-core/tests/integ_tests/async_activity_client_tests.rs +++ b/crates/sdk-core/tests/integ_tests/async_activity_client_tests.rs @@ -135,8 +135,8 @@ async fn async_activity_completions( let err = activity_result.expect_err("expected failure"); if let ActivityExecutionError::Failed(failure) = err { // The failure we sent is wrapped as the cause - let cause = failure.cause.expect("cause should be present"); - assert_eq!(cause.message, "async failure reason"); + let cause = failure.cause().expect("cause should be present"); + assert_eq!(cause.failure().message, "async failure reason"); } else { panic!("expected Failed, got {err:?}"); } diff --git a/crates/sdk-core/tests/integ_tests/data_converter_tests.rs b/crates/sdk-core/tests/integ_tests/data_converter_tests.rs index 7145004d5..335c32c31 100644 --- a/crates/sdk-core/tests/integ_tests/data_converter_tests.rs +++ b/crates/sdk-core/tests/integ_tests/data_converter_tests.rs @@ -8,23 +8,31 @@ use std::{ time::Duration, }; use temporalio_client::{ - Client, ClientOptions, UntypedWorkflow, WorkflowDescribeOptions, WorkflowStartOptions, + Client, ClientOptions, UntypedWorkflow, WorkflowDescribeOptions, WorkflowExecuteUpdateOptions, + WorkflowQueryOptions, WorkflowSignalOptions, WorkflowStartOptions, + errors::{WorkflowGetResultError, WorkflowUpdateError}, }; use temporalio_common::{ data_converters::{ - DataConverter, DefaultFailureConverter, MultiArgs2, PayloadCodec, PayloadConversionError, - PayloadConverter, SerializationContext, SerializationContextData, TemporalDeserializable, - TemporalSerializable, + DataConverter, DefaultFailureConverter, DefaultPayloadCodec, FailureConverter, MultiArgs2, + PayloadCodec, PayloadConversionError, PayloadConverter, SerializationContext, + SerializationContextData, TemporalDeserializable, TemporalSerializable, }, + error::{IncomingError, OutgoingError}, protos::{ coresdk::AsJsonPayloadExt, - temporal::api::{common::v1::Payload, history::v1::history_event::Attributes}, + temporal::api::{ + common::v1::{Payload, RetryPolicy}, + failure::v1::failure::FailureInfo, + history::v1::history_event::Attributes, + }, }, worker::WorkerTaskTypes, }; use temporalio_macros::{activities, workflow, workflow_methods}; use temporalio_sdk::{ - ActivityOptions, WorkflowContext, WorkflowResult, + ActivityOptions, CancellableFuture, SyncWorkflowContext, WorkflowContext, WorkflowContextView, + WorkflowResult, activities::{ActivityContext, ActivityError}, }; @@ -91,6 +99,85 @@ impl TestActivities { } } +struct FailurePayloadActivities; +#[activities] +impl FailurePayloadActivities { + #[activity] + async fn cancel_with_tracked_details(ctx: ActivityContext) -> Result<(), ActivityError> { + let mut ticker = tokio::time::interval(Duration::from_millis(100)); + loop { + tokio::select! { + _ = ctx.cancelled() => break, + _ = ticker.tick() => ctx.record_heartbeat(vec![]), + } + } + Err(ActivityError::cancelled_with_details( + "codec-cancel-details".to_string(), + )) + } + + #[activity] + async fn heartbeat_then_timeout(ctx: ActivityContext) -> Result<(), ActivityError> { + ctx.record_heartbeat(vec![ + TrackedValue::new("codec-heartbeat-details".to_string()) + .as_json_payload() + .map_err(ActivityError::from)?, + ]); + tokio::time::sleep(Duration::from_secs(2)).await; + Ok(()) + } +} + +const FAILURE_CONVERTER_ERROR_MESSAGE: &str = "intentional failure converter error"; +const WORKFLOW_FAILURE_MESSAGE: &str = "workflow converter fallback failure"; +const ACTIVITY_PANIC_MESSAGE: &str = "activity converter fallback panic"; +const QUERY_FAILURE_MESSAGE: &str = "query converter fallback failure"; +const UPDATE_VALIDATOR_FAILURE_MESSAGE: &str = "update validator converter fallback failure"; +const UPDATE_HANDLER_FAILURE_MESSAGE: &str = "update handler converter fallback failure"; + +#[derive(Debug)] +struct FailingFailureConverter; + +impl FailureConverter for FailingFailureConverter { + fn to_failure( + &self, + err: OutgoingError, + _payload_converter: &PayloadConverter, + _context: &SerializationContextData, + ) -> temporalio_common::protos::temporal::api::failure::v1::Failure { + temporalio_common::protos::temporal::api::failure::v1::Failure::application_failure( + format!( + "Failed converting error to failure: Encoding error: {FAILURE_CONVERTER_ERROR_MESSAGE}, original error message: {}", + err + ), + false, + ) + } + + fn to_error( + &self, + failure: temporalio_common::protos::temporal::api::failure::v1::Failure, + payload_converter: &PayloadConverter, + context: &SerializationContextData, + ) -> Result { + DefaultFailureConverter.to_error(failure, payload_converter, context) + } +} + +async fn starter_with_failing_failure_converter(test_name: &str) -> CoreWfStarter { + let connection = get_integ_connection(None).await; + let data_converter = DataConverter::new( + PayloadConverter::default(), + FailingFailureConverter, + DefaultPayloadCodec, + ); + let client_opts = ClientOptions::new(integ_namespace()) + .data_converter(data_converter) + .build(); + let client = Client::new(connection, client_opts).unwrap(); + CoreWfStarter::new_with_overrides(test_name, None, Some(client)) +} + #[workflow] #[derive(Default)] struct DataConverterTestWorkflow; @@ -138,6 +225,394 @@ impl DescribeDataConverterWorkflow { } } +#[workflow] +#[derive(Default)] +struct CancellationDetailsWorkflow; +#[workflow_methods] +impl CancellationDetailsWorkflow { + #[run] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult { + let act = ctx.start_activity( + FailurePayloadActivities::cancel_with_tracked_details, + (), + ActivityOptions::with_start_to_close_timeout(Duration::from_secs(5)) + .heartbeat_timeout(Duration::from_secs(1)) + .cancellation_type( + temporalio_common::protos::coresdk::workflow_commands::ActivityCancellationType::WaitCancellationCompleted, + ) + .build(), + ); + // WaitCancellationCompleted only carries activity-supplied details if the worker has + // started the activity and observed the cancellation. + ctx.timer(Duration::from_millis(100)).await; + act.cancel(); + let err = act.await.expect_err("activity should be cancelled"); + let Some(cancelled) = err.as_cancelled() else { + panic!("expected cancelled failure, got {err:?}"); + }; + let details = cancelled + .details::() + .expect("cancellation details should decode") + .expect("cancellation details should be present"); + Ok(TrackedWrapper(TrackedValue::new(details))) + } +} + +#[workflow] +#[derive(Default)] +struct HeartbeatDetailsWorkflow; +#[workflow_methods] +impl HeartbeatDetailsWorkflow { + #[run] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult { + let err = ctx + .start_activity( + FailurePayloadActivities::heartbeat_then_timeout, + (), + ActivityOptions::with_start_to_close_timeout(Duration::from_secs(5)) + .heartbeat_timeout(Duration::from_secs(1)) + .retry_policy(RetryPolicy { + maximum_attempts: 1, + ..Default::default() + }) + .build(), + ) + .await + .expect_err("activity should time out"); + let timeout = err.as_timeout().expect("activity should timeout"); + let details = timeout + .last_heartbeat_details()? + .expect("heartbeat details should be present"); + Ok(TrackedWrapper(details)) + } +} + +#[workflow] +#[derive(Default)] +struct WorkflowFailureFallbackWorkflow; +#[workflow_methods] +impl WorkflowFailureFallbackWorkflow { + #[run] + async fn run(_ctx: &mut WorkflowContext) -> WorkflowResult<()> { + Err(anyhow::anyhow!(WORKFLOW_FAILURE_MESSAGE).into()) + } +} + +struct PanicActivities; +#[activities] +impl PanicActivities { + #[activity] + async fn panic_activity(_ctx: ActivityContext, _input: String) -> Result<(), ActivityError> { + panic!("{ACTIVITY_PANIC_MESSAGE}"); + } +} + +#[workflow] +#[derive(Default)] +struct ActivityPanicFallbackWorkflow; +#[workflow_methods] +impl ActivityPanicFallbackWorkflow { + #[run] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult<()> { + let _ = ctx + .start_activity( + PanicActivities::panic_activity, + String::new(), + ActivityOptions::with_start_to_close_timeout(Duration::from_secs(5)) + .retry_policy(RetryPolicy { + maximum_attempts: 1, + ..Default::default() + }) + .build(), + ) + .await + .expect_err("activity should fail"); + Ok(()) + } +} + +#[workflow] +#[derive(Default)] +struct QueryUpdateFailureFallbackWorkflow { + finish: bool, +} + +#[workflow_methods] +impl QueryUpdateFailureFallbackWorkflow { + #[run] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult<()> { + ctx.wait_condition(|s| s.finish).await; + Ok(()) + } + + #[signal] + fn finish(&mut self, _ctx: &mut SyncWorkflowContext) { + self.finish = true; + } + + #[query] + fn fail_query( + &self, + _ctx: &WorkflowContextView, + ) -> Result<(), Box> { + Err(QUERY_FAILURE_MESSAGE.into()) + } + + #[update_validator(fail_validated_update)] + fn validate_fail_validated_update( + &self, + _ctx: &WorkflowContextView, + _input: &(), + ) -> Result<(), Box> { + Err(UPDATE_VALIDATOR_FAILURE_MESSAGE.into()) + } + + #[update] + fn fail_validated_update(&mut self, _ctx: &mut SyncWorkflowContext, _input: ()) {} + + #[update] + async fn fail_update( + _ctx: &mut WorkflowContext, + _input: (), + ) -> Result<(), Box> { + Err(UPDATE_HANDLER_FAILURE_MESSAGE.into()) + } +} + +#[tokio::test] +async fn custom_failure_converter_fallback_applied_to_workflow_failures() { + let wf_name = WorkflowFailureFallbackWorkflow::name(); + let mut starter = starter_with_failing_failure_converter(wf_name).await; + starter + .sdk_config + .register_workflow::(); + starter.sdk_config.task_types = WorkerTaskTypes::workflow_only(); + let mut worker = starter.worker().await; + + let task_queue = starter.get_task_queue().to_owned(); + let handle = worker + .submit_workflow( + WorkflowFailureFallbackWorkflow::run, + (), + WorkflowStartOptions::new(task_queue, wf_name.to_owned()).build(), + ) + .await + .unwrap(); + worker.run_until_done().await.unwrap(); + + let failure = match handle.get_result(Default::default()).await.unwrap_err() { + WorkflowGetResultError::Failed(failure) => failure, + err => panic!("unexpected workflow result error: {err:?}"), + }; + assert_eq!( + failure.message, + format!( + "Failed converting error to failure: Encoding error: {FAILURE_CONVERTER_ERROR_MESSAGE}, original error message: {WORKFLOW_FAILURE_MESSAGE}" + ) + ); + assert!(matches!( + failure.failure_info, + Some(FailureInfo::ApplicationFailureInfo(_)) + )); +} + +#[tokio::test] +async fn custom_failure_converter_fallback_applied_to_activity_panic_failures() { + let wf_name = ActivityPanicFallbackWorkflow::name(); + let mut starter = starter_with_failing_failure_converter(wf_name).await; + starter.sdk_config.register_activities(PanicActivities); + starter + .sdk_config + .register_workflow::(); + let mut worker = starter.worker().await; + + let task_queue = starter.get_task_queue().to_owned(); + let handle = worker + .submit_workflow( + ActivityPanicFallbackWorkflow::run, + (), + WorkflowStartOptions::new(task_queue, wf_name.to_owned()).build(), + ) + .await + .unwrap(); + worker.run_until_done().await.unwrap(); + handle.get_result(Default::default()).await.unwrap(); + + let history = handle.fetch_history(Default::default()).await.unwrap(); + let activity_failure = history + .into_events() + .into_iter() + .find_map(|event| match event.attributes { + Some(Attributes::ActivityTaskFailedEventAttributes(attrs)) => attrs.failure, + _ => None, + }) + .expect("workflow history should contain an activity failure"); + + assert_eq!( + activity_failure.message, + format!( + "Failed converting error to failure: Encoding error: {FAILURE_CONVERTER_ERROR_MESSAGE}, original error message: Activity function panicked: {ACTIVITY_PANIC_MESSAGE}" + ) + ); + assert!(matches!( + activity_failure.failure_info, + Some(FailureInfo::ApplicationFailureInfo(_)) + )); +} + +#[tokio::test] +async fn custom_failure_converter_fallback_applied_to_query_failures() { + let wf_name = QueryUpdateFailureFallbackWorkflow::name(); + let mut starter = starter_with_failing_failure_converter(wf_name).await; + starter + .sdk_config + .register_workflow::(); + starter.sdk_config.task_types = WorkerTaskTypes::workflow_only(); + let mut worker = starter.worker().await; + + let task_queue = starter.get_task_queue().to_owned(); + let handle = worker + .submit_workflow( + QueryUpdateFailureFallbackWorkflow::run, + (), + WorkflowStartOptions::new(task_queue, wf_name.to_owned()).build(), + ) + .await + .unwrap(); + + let query_and_finish = async { + let err = handle + .query( + QueryUpdateFailureFallbackWorkflow::fail_query, + (), + WorkflowQueryOptions::default(), + ) + .await + .expect_err("query should fail"); + assert!(err.to_string().contains(&format!( + "Failed converting error to failure: Encoding error: {FAILURE_CONVERTER_ERROR_MESSAGE}, original error message: {QUERY_FAILURE_MESSAGE}" + ))); + handle + .signal( + QueryUpdateFailureFallbackWorkflow::finish, + (), + WorkflowSignalOptions::default(), + ) + .await + .unwrap(); + }; + + let (_, worker_res) = tokio::join!(query_and_finish, worker.run_until_done()); + worker_res.unwrap(); + handle.get_result(Default::default()).await.unwrap(); +} + +#[tokio::test] +async fn custom_failure_converter_fallback_applied_to_update_validation_failures() { + let wf_name = QueryUpdateFailureFallbackWorkflow::name(); + let mut starter = starter_with_failing_failure_converter(wf_name).await; + starter + .sdk_config + .register_workflow::(); + starter.sdk_config.task_types = WorkerTaskTypes::workflow_only(); + let mut worker = starter.worker().await; + + let task_queue = starter.get_task_queue().to_owned(); + let handle = worker + .submit_workflow( + QueryUpdateFailureFallbackWorkflow::run, + (), + WorkflowStartOptions::new(task_queue, format!("{wf_name}_validator")).build(), + ) + .await + .unwrap(); + + let update_and_finish = async { + let err = handle + .execute_update( + QueryUpdateFailureFallbackWorkflow::fail_validated_update, + (), + WorkflowExecuteUpdateOptions::default(), + ) + .await + .expect_err("update should be rejected"); + let WorkflowUpdateError::Failed(failure) = err else { + panic!("expected failed update error"); + }; + assert_eq!( + failure.message, + format!( + "Failed converting error to failure: Encoding error: {FAILURE_CONVERTER_ERROR_MESSAGE}, original error message: {UPDATE_VALIDATOR_FAILURE_MESSAGE}" + ) + ); + handle + .signal( + QueryUpdateFailureFallbackWorkflow::finish, + (), + WorkflowSignalOptions::default(), + ) + .await + .unwrap(); + }; + + let (_, worker_res) = tokio::join!(update_and_finish, worker.run_until_done()); + worker_res.unwrap(); + handle.get_result(Default::default()).await.unwrap(); +} + +#[tokio::test] +async fn custom_failure_converter_fallback_applied_to_update_handler_failures() { + let wf_name = QueryUpdateFailureFallbackWorkflow::name(); + let mut starter = starter_with_failing_failure_converter(wf_name).await; + starter + .sdk_config + .register_workflow::(); + starter.sdk_config.task_types = WorkerTaskTypes::workflow_only(); + let mut worker = starter.worker().await; + + let task_queue = starter.get_task_queue().to_owned(); + let handle = worker + .submit_workflow( + QueryUpdateFailureFallbackWorkflow::run, + (), + WorkflowStartOptions::new(task_queue, format!("{wf_name}_handler")).build(), + ) + .await + .unwrap(); + + let update_and_finish = async { + let err = handle + .execute_update( + QueryUpdateFailureFallbackWorkflow::fail_update, + (), + WorkflowExecuteUpdateOptions::default(), + ) + .await + .expect_err("update should fail"); + let WorkflowUpdateError::Failed(failure) = err else { + panic!("expected failed update error"); + }; + assert_eq!( + failure.message, + format!( + "Failed converting error to failure: Encoding error: {FAILURE_CONVERTER_ERROR_MESSAGE}, original error message: {UPDATE_HANDLER_FAILURE_MESSAGE}" + ) + ); + handle + .signal( + QueryUpdateFailureFallbackWorkflow::finish, + (), + WorkflowSignalOptions::default(), + ) + .await + .unwrap(); + }; + + let (_, worker_res) = tokio::join!(update_and_finish, worker.run_until_done()); + worker_res.unwrap(); + handle.get_result(Default::default()).await.unwrap(); +} + #[tokio::test] async fn data_converter_tracks_serialization_points() { let wf_name = DataConverterTestWorkflow::name(); @@ -547,3 +1022,101 @@ async fn describe_decodes_user_metadata_with_ungated_xor_codec() { assert_eq!(desc.static_summary(), Some("codec summary")); assert_eq!(desc.static_details(), Some("codec details")); } + +#[tokio::test] +async fn codec_roundtrips_activity_cancellation_details() { + let wf_name = CancellationDetailsWorkflow::name(); + let codec = Arc::new(XorCodec::new(0x42)); + + let connection = get_integ_connection(None).await; + let data_converter = DataConverter::new( + PayloadConverter::default(), + DefaultFailureConverter, + codec.clone(), + ); + let client_opts = ClientOptions::new(integ_namespace()) + .data_converter(data_converter) + .build(); + let client = Client::new(connection, client_opts).unwrap(); + + let mut starter = CoreWfStarter::new_with_overrides(wf_name, None, Some(client)); + starter + .sdk_config + .register_activities(FailurePayloadActivities); + starter.sdk_config.task_types = WorkerTaskTypes::all(); + starter + .sdk_config + .register_workflow::(); + let wf_id = starter.get_task_queue().to_owned(); + let mut worker = starter.worker().await; + + let handle = worker + .submit_workflow( + CancellationDetailsWorkflow::run, + (), + WorkflowStartOptions::new(starter.get_task_queue(), wf_id).build(), + ) + .await + .unwrap(); + worker.run_until_done().await.unwrap(); + + let details = handle.get_result(Default::default()).await.unwrap().0; + assert_eq!(details.data, "codec-cancel-details"); + assert!( + codec.encode_count() > 0, + "codec should encode cancellation details" + ); + assert!( + codec.decode_count() > 0, + "codec should decode cancellation details" + ); +} + +#[tokio::test] +async fn codec_roundtrips_activity_heartbeat_timeout_details() { + let wf_name = HeartbeatDetailsWorkflow::name(); + let codec = Arc::new(XorCodec::new(0x42)); + + let connection = get_integ_connection(None).await; + let data_converter = DataConverter::new( + PayloadConverter::default(), + DefaultFailureConverter, + codec.clone(), + ); + let client_opts = ClientOptions::new(integ_namespace()) + .data_converter(data_converter) + .build(); + let client = Client::new(connection, client_opts).unwrap(); + + let mut starter = CoreWfStarter::new_with_overrides(wf_name, None, Some(client)); + starter + .sdk_config + .register_activities(FailurePayloadActivities); + starter.sdk_config.task_types = WorkerTaskTypes::all(); + starter + .sdk_config + .register_workflow::(); + let wf_id = starter.get_task_queue().to_owned(); + let mut worker = starter.worker().await; + + let handle = worker + .submit_workflow( + HeartbeatDetailsWorkflow::run, + (), + WorkflowStartOptions::new(starter.get_task_queue(), wf_id).build(), + ) + .await + .unwrap(); + worker.run_until_done().await.unwrap(); + + let details = handle.get_result(Default::default()).await.unwrap().0; + assert_eq!(details.data, "codec-heartbeat-details"); + assert!( + codec.encode_count() > 0, + "codec should encode heartbeat details" + ); + assert!( + codec.decode_count() > 0, + "codec should decode heartbeat details" + ); +} diff --git a/crates/sdk-core/tests/integ_tests/heartbeat_tests.rs b/crates/sdk-core/tests/integ_tests/heartbeat_tests.rs index 4751e2daf..62e92e372 100644 --- a/crates/sdk-core/tests/integ_tests/heartbeat_tests.rs +++ b/crates/sdk-core/tests/integ_tests/heartbeat_tests.rs @@ -49,11 +49,13 @@ impl ActivityDoesntHeartbeatHitsTimeoutThenCompletesWf { ) .await; let err = res.unwrap_err(); - if let ActivityExecutionError::Failed(f) = &err { - assert_eq!(f.is_timeout(), Some(TimeoutType::Heartbeat)); - } else { + let ActivityExecutionError::Failed(failure) = &err else { panic!("expected Failed, got {err:?}"); - } + }; + let Some(timeout) = failure.as_timeout() else { + panic!("expected timeout cause, got {failure:?}"); + }; + assert_eq!(timeout.timeout_type(), TimeoutType::Heartbeat); Ok(()) } } diff --git a/crates/sdk-core/tests/integ_tests/workflow_tests/activities.rs b/crates/sdk-core/tests/integ_tests/workflow_tests/activities.rs index d1f5b4681..d27daa5ee 100644 --- a/crates/sdk-core/tests/integ_tests/workflow_tests/activities.rs +++ b/crates/sdk-core/tests/integ_tests/workflow_tests/activities.rs @@ -16,32 +16,36 @@ use temporalio_client::{ ActivityIdentifier, UntypedWorkflow, WorkflowDescribeOptions, WorkflowStartOptions, WorkflowTerminateOptions, }; -use temporalio_common::protos::{ - coresdk::{ - ActivityHeartbeat, ActivityTaskCompletion, AsJsonPayloadExt, IntoCompletion, - IntoPayloadsExt, - activity_result::{ - self, ActivityExecutionResult, ActivityResolution, activity_resolution as act_res, - }, - activity_task::activity_task as act_task, - workflow_activation::{ - FireTimer, ResolveActivity, WorkflowActivationJob, workflow_activation_job, + +use temporalio_common::{ + error::{ApplicationFailure, IncomingError}, + protos::{ + coresdk::{ + ActivityHeartbeat, ActivityTaskCompletion, AsJsonPayloadExt, IntoCompletion, + IntoPayloadsExt, + activity_result::{ + self, ActivityExecutionResult, ActivityResolution, activity_resolution as act_res, + }, + activity_task::activity_task as act_task, + workflow_activation::{ + FireTimer, ResolveActivity, WorkflowActivationJob, workflow_activation_job, + }, + workflow_commands::{ + ActivityCancellationType, RequestCancelActivity, ScheduleActivity, StartTimer, + }, + workflow_completion::WorkflowActivationCompletion, }, - workflow_commands::{ - ActivityCancellationType, RequestCancelActivity, ScheduleActivity, StartTimer, + temporal::api::{ + common::v1::{ActivityType, Payload, Payloads, RetryPolicy}, + enums::v1::{CommandType, EventType, RetryState, TimeoutType}, + failure::v1::{ActivityFailureInfo, Failure, failure::FailureInfo}, + sdk::v1::UserMetadata, }, - workflow_completion::WorkflowActivationCompletion, - }, - temporal::api::{ - common::v1::{ActivityType, Payload, Payloads, RetryPolicy}, - enums::v1::{CommandType, EventType, RetryState}, - failure::v1::{ActivityFailureInfo, Failure, failure::FailureInfo}, - sdk::v1::UserMetadata, }, }; use temporalio_macros::{activities, workflow, workflow_methods}; use temporalio_sdk::{ - ActivityOptions, CancellableFuture, WorkflowContext, WorkflowResult, WorkflowTermination, + ActivityExecutionError, ActivityOptions, CancellableFuture, WorkflowContext, WorkflowResult, activities::{ActivityContext, ActivityError}, }; use temporalio_sdk_core::{ @@ -144,6 +148,67 @@ async fn one_activity_only() { assert_eq!(r, input); } +#[tokio::test] +async fn activity_panics_are_retryable() { + struct PanicOnceActivities; + + #[activities] + impl PanicOnceActivities { + #[activity] + async fn panic_once(self: Arc, ctx: ActivityContext) -> Result { + let _ = self; + if ctx.info().attempt == 1 { + panic!("panic once"); + } + Ok(ctx.info().attempt) + } + } + + #[workflow] + #[derive(Default)] + struct ActivityPanicRetryWorkflow; + + #[workflow_methods] + impl ActivityPanicRetryWorkflow { + #[run] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult { + ctx.start_activity( + PanicOnceActivities::panic_once, + (), + ActivityOptions::with_start_to_close_timeout(Duration::from_secs(5)) + .retry_policy(RetryPolicy { + maximum_attempts: 2, + ..Default::default() + }) + .build(), + ) + .await + .map_err(|e| anyhow!("{e}").into()) + } + } + + let wf_name = ActivityPanicRetryWorkflow::name(); + let mut starter = CoreWfStarter::new(wf_name); + starter.sdk_config.register_activities(PanicOnceActivities); + starter + .sdk_config + .register_workflow::(); + let mut worker = starter.worker().await; + + let task_queue = starter.get_task_queue().to_owned(); + let handle = worker + .submit_workflow( + ActivityPanicRetryWorkflow::run, + (), + WorkflowStartOptions::new(task_queue, wf_name.to_owned()).build(), + ) + .await + .unwrap(); + + worker.run_until_done().await.unwrap(); + assert_eq!(handle.get_result(Default::default()).await.unwrap(), 2); +} + #[tokio::test] async fn activity_workflow() { let mut starter = init_core_and_create_wf("activity_workflow").await; @@ -343,6 +408,85 @@ async fn activity_non_retryable_failure_with_error() { core.complete_execution(&task.run_id).await; } +#[tokio::test] +async fn workflow_observes_non_retryable_activity() { + let mut starter = CoreWfStarter::new("workflow-observes-non-retryable-activity-failure"); + starter + .sdk_config + .register_activities(NonRetryableActivityErrorActivities); + let mut worker = starter.worker().await; + + #[workflow] + #[derive(Default)] + struct NonRetryableActivityFailureWorkflow; + + #[workflow_methods] + impl NonRetryableActivityFailureWorkflow { + #[run] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult<()> { + let err = ctx + .start_activity( + NonRetryableActivityErrorActivities::fail_non_retryable, + (), + ActivityOptions::with_start_to_close_timeout(Duration::from_secs(5)) + .activity_id("non-retryable-act".to_owned()) + .retry_policy(RetryPolicy { + maximum_attempts: 3, + ..Default::default() + }) + .build(), + ) + .await + .unwrap_err(); + + let ActivityExecutionError::Failed(fail_err) = err else { + panic!("expected activity failure, got {err:?}"); + }; + assert_eq!(fail_err.activity_id(), "non-retryable-act"); + assert_eq!( + fail_err + .activity_type() + .map(|activity_type| activity_type.name.as_str()), + Some(NonRetryableActivityErrorActivities::fail_non_retryable.name()) + ); + assert_eq!(fail_err.retry_state(), RetryState::NonRetryableFailure); + + let Some(IncomingError::Application(app_err)) = fail_err.cause() else { + panic!("expected application failure cause, got {fail_err:?}"); + }; + assert!(app_err.is_non_retryable()); + assert_eq!(app_err.to_string(), "non-retryable activity failure"); + assert!(app_err.failure().is_some()); + Ok(()) + } + } + + struct NonRetryableActivityErrorActivities; + + #[activities] + impl NonRetryableActivityErrorActivities { + #[activity] + async fn fail_non_retryable(_ctx: ActivityContext) -> Result<(), ActivityError> { + Err(ActivityError::application( + ApplicationFailure::non_retryable(anyhow!("non-retryable activity failure")), + )) + } + } + + worker.register_workflow::(); + + let task_queue = starter.get_task_queue().to_owned(); + worker + .submit_workflow( + NonRetryableActivityFailureWorkflow::run, + (), + WorkflowStartOptions::new(task_queue.clone(), task_queue).build(), + ) + .await + .unwrap(); + worker.run_until_done().await.unwrap(); +} + #[tokio::test] async fn activity_retry() { let mut starter = init_core_and_create_wf("activity_retry").await; @@ -1180,7 +1324,21 @@ async fn activity_can_be_cancelled_by_local_timeout() { .build(), ) .await; - assert!(res.is_err_and(|e| e.is_timeout())); + let err = res.unwrap_err(); + let ActivityExecutionError::Failed(fail_err) = err else { + panic!("expected activity failure, got {err:?}"); + }; + assert_eq!(fail_err.retry_state(), RetryState::MaximumAttemptsReached); + let Some(timeout) = fail_err.as_timeout() else { + panic!("expected timeout cause, got {fail_err:?}"); + }; + assert_eq!(timeout.timeout_type(), TimeoutType::StartToClose); + assert_eq!( + fail_err + .activity_type() + .map(|activity_type| activity_type.name.as_str()), + Some(CancellableEchoActivities::cancellable_echo.name()) + ); Ok(()) } } @@ -1248,7 +1406,7 @@ async fn long_activity_timeout_repro() { ctx.timer(Duration::from_secs(60 * 3)).await; iter += 1; if iter > 5000 { - return Err(WorkflowTermination::continue_as_new(Default::default())); + ctx.continue_as_new(&(), Default::default())?; } } } @@ -1306,8 +1464,7 @@ async fn pass_activity_summary_to_metadata() { .summary("activity summary".to_string()) .build(), ) - .await - .map_err(|e| anyhow!("{e}"))?; + .await?; Ok(()) } } @@ -1371,7 +1528,16 @@ async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [ ctx.timer(Duration::from_secs(1)).await; act_fut.cancel(); ctx.timer(Duration::from_secs(3)).await; - let _ = act_fut.await; + let err = act_fut.await.unwrap_err(); + let ActivityExecutionError::Cancelled(cancel_err) = err else { + panic!("expected cancelled error, got {err:?}"); + }; + assert!(cancel_err.raw_details().is_none()); + assert!( + cancel_err.cause().is_none(), + "expected cancel to be end of cause chain, but found another: {:?}", + cancel_err.cause() + ); Ok(()) } } @@ -1403,7 +1569,12 @@ impl ImmediateActivityCancelationWorkflow { ActivityOptions::start_to_close_timeout(Duration::from_secs(5)), ); cancel_activity_future.cancel(); - let _ = cancel_activity_future.await; + let err = cancel_activity_future.await.unwrap_err(); + let ActivityExecutionError::Cancelled(cancel_err) = err else { + panic!("expected cancelled error, got {err:?}"); + }; + assert!(cancel_err.raw_details().is_none()); + assert!(cancel_err.cause().is_none()); Ok(()) } } diff --git a/crates/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs b/crates/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs index 4e10a2c28..2bf19ce0d 100644 --- a/crates/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +++ b/crates/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs @@ -6,6 +6,7 @@ use temporalio_client::{WorkflowCancelOptions, WorkflowStartOptions}; use temporalio_common::{ UntypedWorkflow, data_converters::RawValue, + error::IncomingError, protos::{ coresdk::{ AsJsonPayloadExt, @@ -33,7 +34,8 @@ use temporalio_common::{ use temporalio_macros::{workflow, workflow_methods}; use temporalio_sdk::{ CancellableFuture, ChildWorkflowExecutionError, ChildWorkflowOptions, ChildWorkflowSignalError, - SyncWorkflowContext, WorkflowContext, WorkflowResult, WorkflowTermination, + ChildWorkflowStartError, SyncWorkflowContext, WorkflowContext, WorkflowResult, + WorkflowTermination, }; use temporalio_sdk_core::{ replay::{DEFAULT_WORKFLOW_TYPE, TestHistoryBuilder, canned_histories}, @@ -477,7 +479,13 @@ impl ParentCancelsChildWf { .result() .await .expect_err("child should be cancelled"); - assert_matches!(err, ChildWorkflowExecutionError::Cancelled(_)); + let ChildWorkflowExecutionError::Failed(failure) = err else { + panic!("started child cancellation should stay wrapper-shaped"); + }; + let Some(cancelled) = failure.as_cancelled() else { + panic!("child failure should retain cancelled reason"); + }; + assert!(cancelled.raw_details().is_none()); Ok(()) } } @@ -490,6 +498,183 @@ async fn cancel_child_workflow() { worker.run().await.unwrap(); } +#[workflow] +#[derive(Default)] +struct RuntimeParentCancelsChildWf; + +#[workflow_methods] +impl RuntimeParentCancelsChildWf { + #[run(name = "runtime_parent_cancels_child")] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult<()> { + let started = ctx + .child_workflow( + GrandchildCancelled::run, + (), + ChildWorkflowOptions { + workflow_id: format!("{}-runtime-cancelled-child", ctx.task_queue()), + cancel_type: ChildWorkflowCancellationType::WaitCancellationCompleted, + ..Default::default() + }, + ) + .await + .expect("child should start"); + started.cancel("child cancel".to_string()); + let err = started + .result() + .await + .expect_err("child should be cancelled"); + let ChildWorkflowExecutionError::Failed(failure) = err else { + panic!("started child cancellation should stay wrapper-shaped"); + }; + let Some(cancelled) = failure.as_cancelled() else { + panic!("child failure should retain cancelled reason"); + }; + assert!(cancelled.raw_details().is_none()); + assert!(cancelled.cause().is_none()); + Ok(()) + } +} + +#[tokio::test] +async fn cancel_child_workflow_runtime_shape() { + let mut starter = CoreWfStarter::new("cancel-child-workflow-runtime-shape"); + starter.sdk_config.task_types = WorkerTaskTypes::workflow_only(); + let mut worker = starter.worker().await; + + worker.register_workflow::(); + worker.register_workflow::(); + + let task_queue = starter.get_task_queue().to_owned(); + worker + .submit_workflow( + RuntimeParentCancelsChildWf::run, + (), + WorkflowStartOptions::new(task_queue.clone(), task_queue).build(), + ) + .await + .unwrap(); + worker.run_until_done().await.unwrap(); +} + +#[workflow] +#[derive(Default)] +struct GrandchildCancelled; + +#[workflow_methods] +impl GrandchildCancelled { + #[run(name = "grandchild_wf")] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult<()> { + ctx.cancelled().await; + Err(WorkflowTermination::Cancelled) + } +} + +#[workflow] +#[derive(Default)] +struct PropagatesChildCancellationWf; + +#[workflow_methods] +impl PropagatesChildCancellationWf { + #[run(name = "child_propagates_cancellation")] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult<()> { + let started = ctx + .child_workflow( + GrandchildCancelled::run, + (), + ChildWorkflowOptions { + workflow_id: format!("{}-grandchild", ctx.task_queue()), + cancel_type: ChildWorkflowCancellationType::WaitCancellationCompleted, + ..Default::default() + }, + ) + .await + .expect("grandchild should start"); + started.cancel("grandchild cancel".to_string()); + started.result().await?; + Ok(()) + } +} + +#[workflow] +#[derive(Default)] +struct GrandchildCancellationWf; + +#[workflow_methods] +impl GrandchildCancellationWf { + #[run(name = "parent_observes_child_failure_from_grandchild_cancellation")] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult<()> { + let child_workflow_id = format!("{}-child", ctx.task_queue()); + let started = ctx + .child_workflow( + PropagatesChildCancellationWf::run, + (), + ChildWorkflowOptions { + workflow_id: child_workflow_id.clone(), + ..Default::default() + }, + ) + .await?; + let err = started.result().await.expect_err("child should fail"); + let ChildWorkflowExecutionError::Failed(failure) = err else { + panic!("child should fail with a child-workflow failure"); + }; + assert_eq!( + failure + .workflow_execution() + .map(|wf| wf.workflow_id.as_str()), + Some(child_workflow_id.as_str()) + ); + assert_eq!( + failure.workflow_type().map(|wf| wf.name.as_str()), + Some("child_propagates_cancellation") + ); + let grandchild_workflow_id = format!("{}-grandchild", ctx.task_queue()); + let Some(IncomingError::ChildWorkflowExecution(grandchild_failure)) = failure.cause() + else { + panic!("child failure should retain the grandchild failure wrapper"); + }; + assert_eq!( + grandchild_failure + .workflow_execution() + .map(|wf| wf.workflow_id.as_str()), + Some(grandchild_workflow_id.as_str()) + ); + assert_eq!( + grandchild_failure + .workflow_type() + .map(|wf| wf.name.as_str()), + Some("grandchild_wf") + ); + let Some(cancelled) = grandchild_failure.as_cancelled() else { + panic!("grandchild failure should retain the cancelled reason"); + }; + assert!(cancelled.raw_details().is_none()); + Ok(()) + } +} + +#[tokio::test] +async fn child_workflow_cancellation_propigates() { + let mut starter = CoreWfStarter::new("child-workflow-cancellation-propigates"); + starter.sdk_config.task_types = WorkerTaskTypes::workflow_only(); + let mut worker = starter.worker().await; + + worker.register_workflow::(); + worker.register_workflow::(); + worker.register_workflow::(); + + let task_queue = starter.get_task_queue().to_owned(); + worker + .submit_workflow( + GrandchildCancellationWf::run, + (), + WorkflowStartOptions::new(task_queue.clone(), task_queue).build(), + ) + .await + .unwrap(); + worker.run_until_done().await.unwrap(); +} + #[rstest::rstest] #[case::abandon(ChildWorkflowCancellationType::Abandon)] #[case::try_cancel(ChildWorkflowCancellationType::TryCancel)] @@ -731,14 +916,26 @@ impl ParentWf { .await; if let Expectation::StartFailure = expectation { match start_res { - Err(ChildWorkflowExecutionError::StartFailed { .. }) => return Ok(()), + Err(ChildWorkflowStartError::StartFailed { .. }) => return Ok(()), _ => return Err(anyhow!("Expected start failure").into()), } } let started = start_res.map_err(|e| anyhow!(e))?; match (expectation, started.result().await) { (Expectation::Success, Ok(_)) => Ok(()), - (Expectation::Failure, Err(ChildWorkflowExecutionError::Failed(_))) => Ok(()), + (Expectation::Failure, Err(ChildWorkflowExecutionError::Failed(failure))) => { + assert_eq!( + failure + .workflow_execution() + .map(|wf| wf.workflow_id.as_str()), + Some("child-id-1") + ); + assert_eq!( + failure.workflow_type().map(|wf| wf.name.as_str()), + Some("child") + ); + Ok(()) + } _ => Err(anyhow!("Unexpected child WF status").into()), } } @@ -841,7 +1038,7 @@ impl CancelBeforeSendWf { ); start.cancel(); match start.await { - Err(ChildWorkflowExecutionError::Cancelled(_)) => Ok(()), + Err(ChildWorkflowStartError::Cancelled(_)) => Ok(()), _ => Err(anyhow!("Unexpected start status").into()), } } @@ -922,6 +1119,52 @@ async fn cancel_child_before_started_event() { .unwrap(); } +#[workflow] +#[derive(Default)] +struct CancelChildBeforeStartedCannedWf; + +#[workflow_methods] +impl CancelChildBeforeStartedCannedWf { + #[run(name = DEFAULT_WORKFLOW_TYPE)] + async fn run(ctx: &mut WorkflowContext) -> WorkflowResult<()> { + let start = ctx.child_workflow( + UntypedWorkflow::new("child"), + RawValue::new(vec![]), + ChildWorkflowOptions { + workflow_id: "child-id-1".to_string(), + cancel_type: ChildWorkflowCancellationType::WaitCancellationCompleted, + ..Default::default() + }, + ); + ctx.cancelled().await; + start.cancel(); + let started = start + .await + .expect("child should still report a successful start"); + let err = started + .result() + .await + .expect_err("child result should be cancelled"); + let ChildWorkflowExecutionError::Failed(failure) = err else { + panic!("started child cancellation should stay wrapper-shaped"); + }; + let Some(cancelled) = failure.as_cancelled() else { + panic!("child failure should retain cancelled reason"); + }; + assert!(cancelled.raw_details().is_none()); + assert!(cancelled.cause().is_none()); + Err(WorkflowTermination::Cancelled) + } +} + +#[tokio::test] +async fn cancel_child_before_started_event_exposes_cancelled_error() { + let t = canned_histories::cancel_child_workflow_before_started_event("child-id-1"); + let mut worker = build_fake_sdk(MockPollCfg::from_resps(t, [ResponseType::AllHistory])); + worker.register_workflow::(); + worker.run().await.unwrap(); +} + #[workflow] struct CancelChildBeforeStartedParent { barr: Arc, @@ -1130,7 +1373,7 @@ impl ChildStartSerializationFailParent { }, ) .await; - assert_matches!(result, Err(ChildWorkflowExecutionError::Serialization(_))); + assert_matches!(result, Err(ChildWorkflowStartError::Serialization(_))); Ok(()) } } diff --git a/crates/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs b/crates/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs index c37948ade..933e405e1 100644 --- a/crates/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +++ b/crates/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs @@ -38,7 +38,7 @@ use temporalio_common::{ command::v1::{RecordMarkerCommandAttributes, command}, common::v1::RetryPolicy, enums::v1::{CommandType, EventType, TimeoutType, WorkflowTaskFailedCause}, - failure::v1::{Failure, failure::FailureInfo}, + failure::v1::Failure, history::v1::history_event::Attributes::MarkerRecordedEventAttributes, query::v1::WorkflowQuery, }, @@ -46,8 +46,9 @@ use temporalio_common::{ }; use temporalio_macros::{activities, workflow, workflow_methods}; use temporalio_sdk::{ - ActivityExecutionError, ActivityOptions, CancellableFuture, LocalActivityOptions, - WorkflowContext, WorkflowContextView, WorkflowResult, WorkflowTermination, + ActivityExecutionError, ActivityOptions, ApplicationFailure, CancellableFuture, + LocalActivityOptions, WorkflowContext, WorkflowContextView, WorkflowResult, + WorkflowTermination, activities::{ActivityContext, ActivityError}, interceptors::{FailOnNondeterminismInterceptor, WorkerInterceptor}, }; @@ -489,7 +490,9 @@ async fn cancel_after_act_starts( tokio::select! { _ = tokio::time::sleep(Duration::from_secs(100)) => {} _ = ctx.cancelled() => { - return Err(ActivityError::cancelled()) + return Err(ActivityError::cancelled_with_details( + "cancel-after-start".to_string(), + )) } _ = self.manual_cancel.cancelled() => { return Ok(()) @@ -546,11 +549,18 @@ async fn cancel_after_act_starts( // This extra timer is here to ensure the presence of another WF task doesn't mess up // resolving the LA with cancel on replay ctx.timer(Duration::from_secs(1)).await; - let resolution = la.await; - assert!(matches!( - resolution, - Err(ActivityExecutionError::Cancelled(_)) - )); + let err = la.await.unwrap_err(); + let ActivityExecutionError::Cancelled(cancel_err) = err else { + panic!("expected cancellation failure, got {err:?}"); + }; + let expected_details = if bo_dur == Duration::from_secs(1) + && cancel_type == ActivityCancellationType::WaitCancellationCompleted + { + Some("cancel-after-start".to_string()) + } else { + None + }; + assert_eq!(cancel_err.details::().unwrap(), expected_details); Ok(()) } } @@ -640,14 +650,11 @@ async fn x_to_close_timeout(#[case] is_schedule: bool) { ) .await; let err = res.unwrap_err(); - if let ActivityExecutionError::Failed(f) = &err { - assert_eq!( - f.is_timeout(), - Some(TimeoutType::try_from(timeout_type).unwrap()) - ); - } else { - return Err(anyhow!("expected Failed, got {err:?}").into()); - } + let timeout = err.as_timeout().unwrap(); + assert_eq!( + timeout.timeout_type(), + TimeoutType::try_from(timeout_type).unwrap() + ); Ok(()) } } @@ -718,11 +725,13 @@ async fn schedule_to_close_timeout_across_timer_backoff(#[case] cached: bool) { ) .await; let err = res.unwrap_err(); - if let ActivityExecutionError::Failed(f) = &err { - assert_eq!(f.is_timeout(), Some(TimeoutType::ScheduleToClose)); - } else { + let ActivityExecutionError::Failed(failure) = &err else { panic!("expected Failed, got {err:?}"); - } + }; + let Some(timeout) = failure.as_timeout() else { + panic!("expected timeout cause, got {failure:?}"); + }; + assert_eq!(timeout.timeout_type(), TimeoutType::ScheduleToClose); Ok(()) } } @@ -1989,11 +1998,14 @@ async fn test_schedule_to_start_timeout() { .await; assert!(la_res.is_err()); if let Err(ActivityExecutionError::Failed(ref fail)) = la_res { - assert_eq!(fail.is_timeout(), Some(TimeoutType::ScheduleToStart)); - assert_matches!(fail.failure_info, Some(FailureInfo::ActivityFailureInfo(_))); - assert_matches!( - fail.cause.as_ref().unwrap().failure_info, - Some(FailureInfo::TimeoutFailureInfo(_)) + let Some(timeout) = fail.as_timeout() else { + panic!("expected timeout cause, got {fail:?}"); + }; + assert_eq!(timeout.timeout_type(), TimeoutType::ScheduleToStart); + assert_eq!( + fail.activity_type() + .map(|activity_type| activity_type.name.as_str()), + Some(StdActivities::echo.name()) ); } Ok(()) @@ -2087,7 +2099,10 @@ async fn test_schedule_to_start_timeout_not_based_on_original_time( if is_sched_to_start { assert!(la_res.is_ok()); } else if let Err(ActivityExecutionError::Failed(ref fail)) = la_res { - assert_eq!(fail.is_timeout(), Some(TimeoutType::ScheduleToClose)); + let Some(timeout) = fail.as_timeout() else { + panic!("expected timeout cause, got {fail:?}"); + }; + assert_eq!(timeout.timeout_type(), TimeoutType::ScheduleToClose); } Ok(()) } @@ -2157,7 +2172,10 @@ async fn start_to_close_timeout_allows_retries(#[values(true, false)] la_complet if la_completes { assert!(la_res.is_ok(), "Result should be ok was {la_res:?}"); } else if let Err(ActivityExecutionError::Failed(ref fail)) = la_res { - assert_eq!(fail.is_timeout(), Some(TimeoutType::StartToClose)); + let Some(timeout) = fail.as_timeout() else { + panic!("expected timeout cause, got {fail:?}"); + }; + assert_eq!(timeout.timeout_type(), TimeoutType::StartToClose); } Ok(()) } @@ -2696,10 +2714,11 @@ async fn local_act_retry_explicit_delay() { // Succeed on 3rd attempt (which is ==2 since fetch_add returns prev val) let last_attempt = self.attempts.fetch_add(1, Ordering::Relaxed); if 0 == last_attempt { - Err(ActivityError::Retryable { - source: anyhow!("Explicit backoff error").into_boxed_dyn_error(), - explicit_delay: Some(Duration::from_millis(300)), - }) + Err(ActivityError::application( + ApplicationFailure::builder(anyhow!("Explicit backoff error")) + .next_retry_delay(Duration::from_millis(300)) + .build(), + )) } else if 2 == last_attempt { Ok(()) } else { @@ -3273,21 +3292,11 @@ async fn cancel_after_act_starts_canned( ctx.timer(Duration::from_secs(1)).await; la.cancel(); ctx.timer(Duration::from_secs(1)).await; - let resolution = la.await; - assert!(matches!( - resolution, - Err(ActivityExecutionError::Cancelled(_)) - )); - if let Err(ActivityExecutionError::Cancelled(rfail)) = resolution { - assert_matches!( - rfail.failure_info, - Some(FailureInfo::ActivityFailureInfo(_)) - ); - assert_matches!( - rfail.cause.unwrap().failure_info, - Some(FailureInfo::CanceledFailureInfo(_)) - ); - } + let err = la.await.unwrap_err(); + let ActivityExecutionError::Cancelled(cancel_err) = err else { + panic!("expected cancelled error, got {err:?}"); + }; + assert!(cancel_err.raw_details().is_none()); Ok(()) } } diff --git a/crates/sdk-core/tests/integ_tests/workflow_tests/nexus.rs b/crates/sdk-core/tests/integ_tests/workflow_tests/nexus.rs index 8a2319e51..8ad0a11e6 100644 --- a/crates/sdk-core/tests/integ_tests/workflow_tests/nexus.rs +++ b/crates/sdk-core/tests/integ_tests/workflow_tests/nexus.rs @@ -498,7 +498,7 @@ async fn nexus_async( Some(nexus_operation_result::Status::Failed(f)) => f ); assert_eq!(f.message, "nexus operation completed unsuccessfully"); - assert_eq!(f.cause.unwrap().message, "Workflow execution error: broken"); + assert_eq!(f.cause.unwrap().message, "broken"); } Outcome::Cancel | Outcome::CancelAfterRecordedBeforeStarted => { let f = assert_matches!( diff --git a/crates/sdk/README.md b/crates/sdk/README.md index f9b4e2e13..16dcd3efe 100644 --- a/crates/sdk/README.md +++ b/crates/sdk/README.md @@ -291,8 +291,7 @@ work. Activities return `Result` with the following error types: -- **`ActivityError::Retryable`** - Transient failure, will be retried -- **`ActivityError::NonRetryable`** - Permanent failure, will not be retried +- **`ActivityError::Application`** - Application failure metadata is carried by `ApplicationFailure` - **`ActivityError::Cancelled`** - Activity was cancelled - **`ActivityError::WillCompleteAsync`** - Activity will complete asynchronously @@ -461,3 +460,18 @@ while let Some(result) = stream.next().await { println!("Workflow: {} ({})", execution.id(), execution.workflow_type()); } ``` + +## Failure Conversion and Error Wrapping + +The default failure converter preserves Temporal failure types when errors cross workflow or +activity boundaries. + +This matters when Rust error propagation wraps Temporal SDK error types, e.g. `anyhow::Error`. +When an `ApplicationFailure` is created from an error whose source is a known Temporal SDK error, the +converter skips the outer error for the failure cause and encodes the known Temporal error +directly. The application failure's own message and metadata are still preserved. + +This keeps the Rust SDK's `Failure`s aligned with other Temporal SDKs: SDK error types +remain represented as Temporal failure types, while unknown Rust error types are encoded as +application failures. + diff --git a/crates/sdk/examples/saga/workflows.rs b/crates/sdk/examples/saga/workflows.rs index bd87244d6..408ca7bc2 100644 --- a/crates/sdk/examples/saga/workflows.rs +++ b/crates/sdk/examples/saga/workflows.rs @@ -2,7 +2,7 @@ use std::time::Duration; use temporalio_macros::{activities, workflow, workflow_methods}; use temporalio_sdk::{ - ActivityExecutionError, ActivityOptions, WorkflowContext, WorkflowResult, WorkflowTermination, + ActivityExecutionError, ActivityOptions, ApplicationFailure, WorkflowContext, WorkflowResult, activities::{ActivityContext, ActivityError}, }; @@ -30,7 +30,7 @@ impl SagaWorkflow { Ok(()) => Ok(compensations.into_iter().map(|(_, id)| id).collect()), Err(e) => { Self::run_compensations(ctx, &compensations).await; - Err(WorkflowTermination::failed(e)) + Err(e.into()) } } } @@ -132,8 +132,10 @@ impl BookingActivities { #[activity] pub async fn book_car(_ctx: ActivityContext, trip_id: String) -> Result { if trip_id.contains("fail") { - return Err(ActivityError::NonRetryable( - anyhow::anyhow!("Car booking failed for trip {trip_id}").into(), + return Err(ActivityError::application( + ApplicationFailure::non_retryable(anyhow::anyhow!( + "Car booking failed for trip {trip_id}" + )), )); } Ok(format!("car-{trip_id}")) diff --git a/crates/sdk/src/activities.rs b/crates/sdk/src/activities.rs index 6a69c53fe..c35e7ca09 100644 --- a/crates/sdk/src/activities.rs +++ b/crates/sdk/src/activities.rs @@ -61,6 +61,7 @@ use temporalio_common::{ data_converters::{ DataConverter, GenericPayloadConverter, SerializationContext, SerializationContextData, }, + error::{ApplicationFailure, FailurePayloads}, protos::{ coresdk::{ActivityHeartbeat, activity_task}, temporal::api::common::v1::{Payload, RetryPolicy, WorkflowExecution}, @@ -229,22 +230,13 @@ pub struct ActivityInfo { /// Returned as errors from activity functions. #[derive(Debug)] pub enum ActivityError { - /// This error can be returned from activities to allow the explicit configuration of certain - /// error properties. It's also the default error type that arbitrary errors will be converted - /// into. - Retryable { - /// The underlying error - source: Box, - /// If specified, the next retry (if there is one) will occur after this delay - explicit_delay: Option, - }, + /// Return this error to attach application-failure metadata to an activity failure. + Application(Box), /// Return this error to indicate your activity is cancelling Cancelled { - /// Some data to save as the cancellation reason - details: Option, + /// Optional cancellation details. + details: Option, }, - /// Return this error to indicate that the activity should not be retried. - NonRetryable(Box), /// Return this error to indicate that the activity will be completed outside of this activity /// definition, by an external client. WillCompleteAsync, @@ -255,6 +247,22 @@ impl ActivityError { pub fn cancelled() -> Self { Self::Cancelled { details: None } } + + /// Construct a cancelled error with details that will be converted using the active data + /// converter. + pub fn cancelled_with_details(details: T) -> Self + where + T: Into, + { + Self::Cancelled { + details: Some(details.into()), + } + } + + /// Construct an application activity error. + pub fn application(err: ApplicationFailure) -> Self { + Self::Application(err.into()) + } } impl From for ActivityError @@ -262,9 +270,9 @@ where E: Into, { fn from(source: E) -> Self { - Self::Retryable { - source: source.into().into_boxed_dyn_error(), - explicit_delay: None, + match source.into().downcast::() { + Ok(application_failure) => Self::Application(Box::new(application_failure)), + Err(err) => Self::Application(ApplicationFailure::new(err).into()), } } } @@ -419,3 +427,52 @@ impl Debug for ActivityDefinitions { .finish() } } + +#[cfg(test)] +mod test { + use super::*; + use rstest::rstest; + + #[rstest] + #[case(true)] + #[case(false)] + fn activity_error_conversion_is_not_lossy(#[case] non_retryable: bool) { + use temporalio_common::protos::temporal::api::enums::v1::ApplicationErrorCategory; + + let original = ApplicationFailure::builder(anyhow::anyhow!("big boom")) + .type_name("BigBoom".to_owned()) + .non_retryable(non_retryable) + .next_retry_delay(StdDuration::from_secs(3)) + .category(ApplicationErrorCategory::Benign) + .details("details") + .build(); + let err = ActivityError::from(original); + let ActivityError::Application(actual) = err else { + panic!("application failure should become app failure") + }; + assert_eq!(actual.type_name(), Some("BigBoom")); + assert_eq!(actual.is_non_retryable(), non_retryable); + assert_eq!(actual.next_retry_delay(), Some(StdDuration::from_secs(3))); + assert_eq!(actual.category(), ApplicationErrorCategory::Benign); + assert_eq!(actual.to_string(), "big boom"); + } + + #[test] + fn activity_error_from_special_err_becomes_application() { + #[derive(Debug, PartialEq)] + struct MyError; + + impl std::error::Error for MyError {} + impl std::fmt::Display for MyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("MyError") + } + } + + let err = ActivityError::from(MyError); + let ActivityError::Application(actual) = err else { + panic!("expected application failure, got {err:?}") + }; + assert_eq!(actual.to_string(), "MyError"); + } +} diff --git a/crates/sdk/src/error.rs b/crates/sdk/src/error.rs new file mode 100644 index 000000000..7b9d5e270 --- /dev/null +++ b/crates/sdk/src/error.rs @@ -0,0 +1,7 @@ +//! Shared SDK error re-exports. + +pub use temporalio_common::error::{ + ActivityExecutionError, ApplicationFailure, ChildWorkflowExecutionError, + ChildWorkflowSignalError, ChildWorkflowStartError, OutgoingActivityError, OutgoingError, + OutgoingWorkflowError, +}; diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index 6805476cb..f979771bf 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -72,6 +72,7 @@ extern crate tracing; extern crate self as temporalio_sdk; pub mod activities; +pub mod error; pub mod interceptors; mod workflow_context; mod workflow_executor; @@ -96,12 +97,16 @@ macro_rules! __temporal_join { use workflow_future::WorkflowFunction; +pub use error::{ + ActivityExecutionError, ApplicationFailure, ChildWorkflowExecutionError, + ChildWorkflowSignalError, ChildWorkflowStartError, OutgoingActivityError, OutgoingError, + OutgoingWorkflowError, +}; pub use temporalio_client::Namespace; pub use workflow_context::{ - ActivityCloseTimeouts, ActivityExecutionError, ActivityOptions, BaseWorkflowContext, - CancellableFuture, ChildWorkflowExecutionError, ChildWorkflowOptions, ChildWorkflowSignalError, - ContinueAsNewOptions, ExternalWorkflowHandle, LocalActivityOptions, NexusOperationOptions, - ParentWorkflowInfo, RootWorkflowInfo, Signal, SignalData, + ActivityCloseTimeouts, ActivityOptions, BaseWorkflowContext, CancellableFuture, + ChildWorkflowOptions, ContinueAsNewOptions, ExternalWorkflowHandle, LocalActivityOptions, + NexusOperationOptions, ParentWorkflowInfo, RootWorkflowInfo, Signal, SignalData, StartChildWorkflowExecutionFailedCause, StartedChildWorkflow, SyncWorkflowContext, TimerOptions, WorkflowContext, WorkflowContextView, }; @@ -155,9 +160,7 @@ use temporalio_common::{ workflow_completion::WorkflowActivationCompletion, }, temporal::api::{ - common::v1::Payload, - enums::v1::WorkflowTaskFailedCause, - failure::v1::{Failure, failure}, + common::v1::Payload, enums::v1::WorkflowTaskFailedCause, failure::v1::Failure, }, }, worker::{WorkerDeploymentOptions, WorkerTaskTypes, build_id_from_current_exe}, @@ -615,13 +618,13 @@ impl Worker { }| { let wf_half = &*wf_half; async move { - let result = join_handle.await.map_err(|e| anyhow::anyhow!("{e}"))?; + let result = join_handle.await.map_err(anyhow::Error::new)?; // Eviction is normal workflow lifecycle - workflows loop waiting for // eviction after completion to manage cache cleanup if let Err(e) = result && !matches!(e, WorkflowTermination::Evicted) { - return Err(e.into()); + return Err(anyhow::Error::new(e)); } debug!(run_id=%run_id, "Removing workflow from cache"); wf_half.workflows.borrow_mut().remove(&run_id); @@ -804,7 +807,6 @@ impl WorkflowHalf { _ => None, }) { let workflow_type = sw.workflow_type.clone(); - let payload_converter = common.data_converter.payload_converter().clone(); let (wff, activations) = { if let Some(factory) = self.workflow_definitions.get_workflow(&workflow_type) { match WorkflowFunction::from_invocation(factory).start_workflow( @@ -813,7 +815,7 @@ impl WorkflowHalf { run_id.clone(), std::mem::take(sw), completions_tx.clone(), - payload_converter, + common.data_converter.clone(), self.detect_nondeterministic_futures, ) { Ok(result) => result, @@ -940,6 +942,7 @@ impl ActivityHalf { let (ctx, args) = ActivityContext::new(worker.clone(), ct, task_queue, task_token.clone(), start); + let activity_data_converter = data_converter.clone(); let codec_data_converter = data_converter.clone(); tokio::spawn(async move { @@ -949,42 +952,42 @@ impl ActivityHalf { .record("temporalWorkflowID", &info.workflow_id) .record("temporalRunID", &info.run_id); } - (act_fn)(args, data_converter, ctx).await + (act_fn)(args, activity_data_converter, ctx).await } .instrument(span); let output = AssertUnwindSafe(act_fut).catch_unwind().await; + let activity_context = SerializationContextData::Activity; let result = match output { - Err(e) => ActivityExecutionResult::fail(Failure::application_failure( - format!("Activity function panicked: {}", panic_formatter(e)), - true, - )), + Err(e) => ActivityExecutionResult::fail( + data_converter.to_failure( + &activity_context, + OutgoingError::Activity(OutgoingActivityError::Application( + ApplicationFailure::new(anyhow!( + "Activity function panicked: {}", + panic_formatter(e) + )) + .into(), + )), + ), + ), Ok(Ok(p)) => ActivityExecutionResult::ok(p), Ok(Err(err)) => match err { - ActivityError::Retryable { - source, - explicit_delay, - } => ActivityExecutionResult::fail({ - let mut f = Failure::application_failure_from_error( - anyhow::Error::from_boxed(source), - false, - ); - if let Some(d) = explicit_delay - && let Some(failure::FailureInfo::ApplicationFailureInfo(fi)) = - f.failure_info.as_mut() - { - fi.next_retry_delay = d.try_into().ok(); - } - f - }), + ActivityError::Application(app) => { + ActivityExecutionResult::fail(data_converter.to_failure( + &activity_context, + OutgoingError::Activity(OutgoingActivityError::Application( + app, + )), + )) + } ActivityError::Cancelled { details } => { - ActivityExecutionResult::cancel_from_details(details) + ActivityExecutionResult::cancel(data_converter.to_failure( + &activity_context, + OutgoingError::Activity(OutgoingActivityError::Cancelled { + details, + }), + )) } - ActivityError::NonRetryable(nre) => ActivityExecutionResult::fail( - Failure::application_failure_from_error( - anyhow::Error::from_boxed(nre), - true, - ), - ), ActivityError::WillCompleteAsync => { ActivityExecutionResult::will_complete_async() } @@ -1251,7 +1254,7 @@ pub enum WorkflowTermination { /// The workflow failed with an error. #[error("Workflow failed: {0}")] - Failed(#[source] anyhow::Error), + Failed(#[source] OutgoingWorkflowError), } impl WorkflowTermination { @@ -1260,33 +1263,45 @@ impl WorkflowTermination { Self::ContinueAsNew(Box::new(can)) } - /// Construct a [WorkflowTermination::Failed] variant from any error. - pub fn failed(err: impl Into) -> Self { + /// Construct a [WorkflowTermination::Failed] variant from an application failure. + pub fn failed_application(err: ApplicationFailure) -> Self { Self::Failed(err.into()) } } impl From for WorkflowTermination { fn from(err: anyhow::Error) -> Self { - Self::Failed(err) + Self::Failed(err.into()) + } +} + +impl From for WorkflowTermination { + fn from(err: temporalio_common::data_converters::PayloadConversionError) -> Self { + Self::Failed(err.into()) } } impl From for WorkflowTermination { fn from(value: ActivityExecutionError) -> Self { - Self::failed(value) + Self::Failed(value.into()) } } impl From for WorkflowTermination { fn from(value: ChildWorkflowExecutionError) -> Self { - Self::failed(value) + Self::Failed(value.into()) + } +} + +impl From for WorkflowTermination { + fn from(value: ChildWorkflowStartError) -> Self { + Self::Failed(value.into()) } } impl From for WorkflowTermination { fn from(value: ChildWorkflowSignalError) -> Self { - Self::failed(value) + Self::Failed(value.into()) } } @@ -1325,6 +1340,7 @@ fn _panic_formatter(panic: Box) -> Box trait PrintablePanicType: Display { type NextType: PrintablePanicType; } + impl PrintablePanicType for &str { type NextType = String; } diff --git a/crates/sdk/src/workflow_context.rs b/crates/sdk/src/workflow_context.rs index 57afa51ec..a77b40a4f 100644 --- a/crates/sdk/src/workflow_context.rs +++ b/crates/sdk/src/workflow_context.rs @@ -35,12 +35,18 @@ use std::{ use temporalio_common::{ ActivityDefinition, SignalDefinition, WorkflowDefinition, data_converters::{ + ActivityExecutionDecodeHint, ChildWorkflowExecutionDecodeHint, + ChildWorkflowSignalDecodeHint, ChildWorkflowStartDecodeHint, DataConverter, GenericPayloadConverter, PayloadConversionError, PayloadConverter, SerializationContext, SerializationContextData, TemporalDeserializable, }, + error::{ + ActivityExecutionError, ChildWorkflowExecutionError, ChildWorkflowSignalError, + ChildWorkflowStartError, + }, protos::{ coresdk::{ - activity_result::{ActivityResolution, activity_resolution}, + activity_result::{ActivityResolution, Cancellation, activity_resolution}, child_workflow::ChildWorkflowResult, common::NamespacedWorkflowExecution, nexus::NexusOperationResult, @@ -57,7 +63,7 @@ use temporalio_common::{ }, temporal::api::{ common::v1::{Memo, Payload, SearchAttributes}, - failure::v1::Failure, + failure::v1::{CanceledFailureInfo, Failure, failure::FailureInfo}, sdk::v1::UserMetadata, }, }, @@ -97,7 +103,7 @@ struct WorkflowContextInner { am_cancelled: watch::Receiver>, shared: RefCell, seq_nums: RefCell, - payload_converter: PayloadConverter, + data_converter: DataConverter, state_mutated: Cell, } @@ -278,68 +284,6 @@ impl WorkflowContextView { } } -/// Error type for activity execution outcomes. -#[derive(Debug, thiserror::Error)] -pub enum ActivityExecutionError { - /// The activity failed with the given failure details. - #[error("Activity failed: {}", .0.message)] - Failed(Box), - /// The activity was cancelled. - #[error("Activity cancelled: {}", .0.message)] - Cancelled(Box), - // TODO: Timed out variant - /// Failed to serialize input or deserialize result payload. - #[error("Payload conversion failed: {0}")] - Serialization(#[from] PayloadConversionError), -} - -impl ActivityExecutionError { - /// Returns true if this error represents a timeout. - pub fn is_timeout(&self) -> bool { - match self { - ActivityExecutionError::Failed(f) => f.is_timeout().is_some(), - _ => false, - } - } -} - -/// Error returned when a child workflow execution fails. -#[derive(Debug, thiserror::Error)] -pub enum ChildWorkflowExecutionError { - /// The child workflow failed. - #[error("Child workflow failed: {}", .0.message)] - Failed(Box), - /// The child workflow was cancelled. - #[error("Child workflow cancelled: {}", .0.message)] - Cancelled(Box), - /// The child workflow failed to start (e.g., workflow ID already exists). - #[error( - "Child workflow start failed: workflow_id={workflow_id}, workflow_type={workflow_type}, cause={cause:?}" - )] - StartFailed { - /// The workflow ID that was requested. - workflow_id: String, - /// The workflow type that was requested. - workflow_type: String, - /// The cause of the start failure. - cause: StartChildWorkflowExecutionFailedCause, - }, - /// Failed to serialize input or deserialize the child workflow result payload. - #[error("Payload conversion failed: {0}")] - Serialization(#[from] PayloadConversionError), -} - -/// Error returned when signaling a child workflow fails. -#[derive(Debug, thiserror::Error)] -pub enum ChildWorkflowSignalError { - /// The signal delivery failed. - #[error("Child workflow signal failed: {}", .0.message)] - Failed(Box), - /// Failed to serialize the signal input payload. - #[error("Signal payload conversion failed: {0}")] - Serialization(#[from] PayloadConversionError), -} - impl BaseWorkflowContext { /// Create a new base context, returning the context itself and a receiver which outputs commands /// sent from the workflow. @@ -349,7 +293,7 @@ impl BaseWorkflowContext { run_id: String, init_workflow_job: InitializeWorkflow, am_cancelled: watch::Receiver>, - payload_converter: PayloadConverter, + data_converter: DataConverter, ) -> (Self, Receiver) { // The receiving side is non-async let (chan, rx) = std::sync::mpsc::channel(); @@ -378,7 +322,7 @@ impl BaseWorkflowContext { next_signal_external_wf_sequence_number: 1, next_nexus_op_sequence_number: 1, }), - payload_converter, + data_converter, state_mutated: Cell::new(false), }), }, @@ -467,11 +411,12 @@ impl BaseWorkflowContext { AD::Output: TemporalDeserializable, { let input = input.into(); + let payload_converter = self.inner.data_converter.payload_converter(); let ctx = SerializationContext { data: &SerializationContextData::Workflow, - converter: &self.inner.payload_converter, + converter: payload_converter, }; - let payloads = match self.inner.payload_converter.to_payloads(&ctx, &input) { + let payloads = match payload_converter.to_payloads(&ctx, &input) { Ok(p) => p, Err(e) => { return ActivityFut::eager(e.into()); @@ -490,7 +435,7 @@ impl BaseWorkflowContext { } .into(), ); - ActivityFut::running(cmd, self.inner.payload_converter.clone()) + ActivityFut::running(cmd, self.inner.data_converter.clone()) } /// Request to run a local activity @@ -504,11 +449,12 @@ impl BaseWorkflowContext { AD::Output: TemporalDeserializable, { let input = input.into(); + let payload_converter = self.inner.data_converter.payload_converter(); let ctx = SerializationContext { data: &SerializationContextData::Workflow, - converter: &self.inner.payload_converter, + converter: payload_converter, }; - let payloads = match self.inner.payload_converter.to_payloads(&ctx, &input) { + let payloads = match payload_converter.to_payloads(&ctx, &input) { Ok(p) => p, Err(e) => { return ActivityFut::eager(e.into()); @@ -516,7 +462,7 @@ impl BaseWorkflowContext { }; ActivityFut::running( LATimerBackoffFut::new(AD::name().to_string(), payloads, opts, self.clone()), - self.inner.payload_converter.clone(), + self.inner.data_converter.clone(), ) } @@ -526,16 +472,17 @@ impl BaseWorkflowContext { workflow: WD, input: impl Into, opts: ChildWorkflowOptions, - ) -> impl CancellableFutureWithReason, ChildWorkflowExecutionError>> + ) -> impl CancellableFutureWithReason, ChildWorkflowStartError>> where WD::Output: TemporalDeserializable, { let input = input.into(); + let payload_converter = self.inner.data_converter.payload_converter(); let ctx = SerializationContext { data: &SerializationContextData::Workflow, - converter: &self.inner.payload_converter, + converter: payload_converter, }; - let payloads = match self.inner.payload_converter.to_payloads(&ctx, &input) { + let payloads = match payload_converter.to_payloads(&ctx, &input) { Ok(p) => p, Err(e) => { return ChildWorkflowStartFut::eager(e.into()); @@ -564,7 +511,7 @@ impl BaseWorkflowContext { child_seq, result_future: result_cmd, base_ctx: self.clone(), - payload_converter: self.inner.payload_converter.clone(), + data_converter: self.inner.data_converter.clone(), }; let (cmd, unblocker) = CancellableWFCommandFut::new_with_dat( @@ -714,7 +661,7 @@ impl SyncWorkflowContext { /// Returns the [PayloadConverter] currently used by the worker running this workflow. pub fn payload_converter(&self) -> &PayloadConverter { - &self.base.inner.payload_converter + self.base.inner.data_converter.payload_converter() } /// Return various information that the workflow was initialized with. Will eventually become @@ -752,14 +699,14 @@ impl SyncWorkflowContext { where W: crate::workflows::WorkflowImplementation, { - let pc = &self.base.inner.payload_converter; + let pc = self.base.inner.data_converter.payload_converter(); let ctx = SerializationContext { data: &SerializationContextData::Workflow, converter: pc, }; let arguments = pc .to_payloads(&ctx, input) - .map_err(WorkflowTermination::failed)?; + .map_err(WorkflowTermination::from)?; let workflow_type = self.workflow_initial_info().workflow_type.clone(); let proto = opts.into_proto(workflow_type, arguments); Err(WorkflowTermination::continue_as_new(proto)) @@ -803,7 +750,7 @@ impl SyncWorkflowContext { workflow: WD, input: impl Into, opts: ChildWorkflowOptions, - ) -> impl CancellableFutureWithReason, ChildWorkflowExecutionError>> + ) -> impl CancellableFutureWithReason, ChildWorkflowStartError>> where WD::Output: TemporalDeserializable, { @@ -1081,7 +1028,7 @@ impl WorkflowContext { workflow: WD, input: impl Into, opts: ChildWorkflowOptions, - ) -> impl CancellableFutureWithReason, ChildWorkflowExecutionError>> + ) -> impl CancellableFutureWithReason, ChildWorkflowStartError>> where WD::Output: TemporalDeserializable, { @@ -1458,9 +1405,15 @@ impl Future for LATimerBackoffFut { } else { self.terminated = true; Poll::Ready(ActivityResolution { - status: Some( - activity_resolution::Status::Cancelled(Default::default()), - ), + status: Some(activity_resolution::Status::Cancelled(Cancellation { + failure: Some(Failure { + message: "Activity cancelled".to_owned(), + failure_info: Some(FailureInfo::CanceledFailureInfo( + CanceledFailureInfo::default(), + )), + ..Default::default() + }), + })), }) } } @@ -1477,7 +1430,15 @@ impl Future for LATimerBackoffFut { if self.did_cancel.load(Ordering::Acquire) { self.terminated = true; return Poll::Ready(ActivityResolution { - status: Some(activity_resolution::Status::Cancelled(Default::default())), + status: Some(activity_resolution::Status::Cancelled(Cancellation { + failure: Some(Failure { + message: "Activity cancelled".to_owned(), + failure_info: Some(FailureInfo::CanceledFailureInfo( + CanceledFailureInfo::default(), + )), + ..Default::default() + }), + })), }); } @@ -1517,13 +1478,13 @@ impl CancellableFuture for LATimerBackoffFut { enum ActivityFut { /// Immediate error (e.g., input serialization failure). Resolves on first poll. Errored { - error: Option, + error: Option>, _phantom: PhantomData, }, /// Running activity that will deserialize output on completion. Running { inner: F, - payload_converter: PayloadConverter, + data_converter: DataConverter, _phantom: PhantomData, }, Terminated, @@ -1532,15 +1493,15 @@ enum ActivityFut { impl ActivityFut { fn eager(err: ActivityExecutionError) -> Self { Self::Errored { - error: Some(err), + error: Some(Box::new(err)), _phantom: PhantomData, } } - fn running(inner: F, payload_converter: PayloadConverter) -> Self { + fn running(inner: F, data_converter: DataConverter) -> Self { Self::Running { inner, - payload_converter, + data_converter, _phantom: PhantomData, } } @@ -1559,20 +1520,26 @@ where let this = self.get_mut(); let poll = match this { ActivityFut::Errored { error, .. } => { - Poll::Ready(Err(error.take().expect("polled after completion"))) + Poll::Ready(Err(*error.take().expect("polled after completion"))) } ActivityFut::Running { inner, - payload_converter, + data_converter, .. } => match Pin::new(inner).poll(cx) { Poll::Pending => Poll::Pending, Poll::Ready(resolution) => Poll::Ready({ let status = resolution.status.ok_or_else(|| { - ActivityExecutionError::Failed(Box::new(Failure { - message: "Activity completed without a status".to_string(), - ..Default::default() - })) + data_converter + .to_error( + &SerializationContextData::Workflow, + Failure { + message: "Activity completed without a status".to_string(), + ..Default::default() + }, + ActivityExecutionDecodeHint { cancelled: false }, + ) + .expect("synthetic activity failure should decode") })?; match status { @@ -1580,20 +1547,23 @@ where let payload = success.result.unwrap_or_default(); let ctx = SerializationContext { data: &SerializationContextData::Workflow, - converter: payload_converter, + converter: data_converter.payload_converter(), }; - payload_converter + data_converter + .payload_converter() .from_payload::(&ctx, payload) .map_err(ActivityExecutionError::Serialization) } - activity_resolution::Status::Failed(f) => Err( - ActivityExecutionError::Failed(Box::new(f.failure.unwrap_or_default())), - ), - activity_resolution::Status::Cancelled(c) => { - Err(ActivityExecutionError::Cancelled(Box::new( - c.failure.unwrap_or_default(), - ))) - } + activity_resolution::Status::Failed(f) => Err(data_converter.to_error( + &SerializationContextData::Workflow, + f.failure.unwrap_or_default(), + ActivityExecutionDecodeHint { cancelled: false }, + )?), + activity_resolution::Status::Cancelled(c) => Err(data_converter.to_error( + &SerializationContextData::Workflow, + c.failure.unwrap_or_default(), + ActivityExecutionDecodeHint { cancelled: true }, + )?), activity_resolution::Status::Backoff(_) => { panic!("DoBackoff should be handled by LATimerBackoffFut") } @@ -1636,7 +1606,7 @@ pub(crate) struct ChildWfCommon { child_seq: u32, result_future: CancellableWFCommandFut, base_ctx: BaseWorkflowContext, - payload_converter: PayloadConverter, + data_converter: DataConverter, } /// Child workflow in pending state. Internal type used during the start handshake; @@ -1665,7 +1635,7 @@ pub struct StartedChildWorkflow { enum ChildWorkflowFut { Running { inner: F, - payload_converter: PayloadConverter, + data_converter: DataConverter, _phantom: PhantomData, }, Terminated, @@ -1685,39 +1655,48 @@ where let poll = match this { ChildWorkflowFut::Running { inner, - payload_converter, + data_converter, .. } => match Pin::new(inner).poll(cx) { Poll::Pending => Poll::Pending, Poll::Ready(result) => Poll::Ready({ use temporalio_common::protos::coresdk::child_workflow::child_workflow_result; let status = result.status.ok_or_else(|| { - ChildWorkflowExecutionError::Failed(Box::new(Failure { - message: "Child workflow completed without a status".to_string(), - ..Default::default() - })) + data_converter + .to_error( + &SerializationContextData::Workflow, + Failure { + message: "Child workflow completed without a status" + .to_string(), + ..Default::default() + }, + ChildWorkflowExecutionDecodeHint, + ) + .expect("synthetic child workflow failure should decode") })?; match status { child_workflow_result::Status::Completed(success) => { let payloads = success.result.into_iter().collect(); let ctx = SerializationContext { data: &SerializationContextData::Workflow, - converter: payload_converter, + converter: data_converter.payload_converter(), }; - payload_converter + data_converter + .payload_converter() .from_payloads::(&ctx, payloads) .map_err(ChildWorkflowExecutionError::Serialization) } - child_workflow_result::Status::Failed(f) => { - Err(ChildWorkflowExecutionError::Failed(Box::new( - f.failure.unwrap_or_default(), - ))) - } - child_workflow_result::Status::Cancelled(c) => { - Err(ChildWorkflowExecutionError::Cancelled(Box::new( + child_workflow_result::Status::Failed(f) => Err(data_converter.to_error( + &SerializationContextData::Workflow, + f.failure.unwrap_or_default(), + ChildWorkflowExecutionDecodeHint, + )?), + child_workflow_result::Status::Cancelled(c) => Err(data_converter + .to_error( + &SerializationContextData::Workflow, c.failure.unwrap_or_default(), - ))) - } + ChildWorkflowExecutionDecodeHint, + )?), } }), }, @@ -1771,7 +1750,7 @@ where enum ChildWorkflowStartFut { /// Immediate error (e.g., input serialization failure). Resolves on first poll. Errored { - error: Option, + error: Option>, _phantom: PhantomData, }, Running(F), @@ -1779,9 +1758,9 @@ enum ChildWorkflowStartFut { } impl ChildWorkflowStartFut { - fn eager(err: ChildWorkflowExecutionError) -> Self { + fn eager(err: ChildWorkflowStartError) -> Self { Self::Errored { - error: Some(err), + error: Some(Box::new(err)), _phantom: PhantomData, } } @@ -1794,13 +1773,13 @@ where F: Future> + Unpin, WD: WorkflowDefinition, { - type Output = Result, ChildWorkflowExecutionError>; + type Output = Result, ChildWorkflowStartError>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); let poll = match this { ChildWorkflowStartFut::Errored { error, .. } => { - Poll::Ready(Err(error.take().expect("polled after completion"))) + Poll::Ready(Err(*error.take().expect("polled after completion"))) } ChildWorkflowStartFut::Running(inner) => match Pin::new(inner).poll(cx) { Poll::Pending => Poll::Pending, @@ -1811,7 +1790,7 @@ where _phantom: PhantomData, }), ChildWorkflowStartStatus::Failed(f) => { - Err(ChildWorkflowExecutionError::StartFailed { + Err(ChildWorkflowStartError::StartFailed { workflow_id: f.workflow_id, workflow_type: f.workflow_type, cause: StartChildWorkflowExecutionFailedCause::try_from(f.cause) @@ -1819,9 +1798,11 @@ where }) } ChildWorkflowStartStatus::Cancelled(c) => { - Err(ChildWorkflowExecutionError::Cancelled(Box::new( + Err(pending.common.data_converter.to_error( + &SerializationContextData::Workflow, c.failure.unwrap_or_default(), - ))) + ChildWorkflowStartDecodeHint, + )?) } }), }, @@ -1844,7 +1825,7 @@ where } } -impl CancellableFuture, ChildWorkflowExecutionError>> +impl CancellableFuture, ChildWorkflowStartError>> for ChildWorkflowStartFut where F: CancellableFutureWithReason> + Unpin, @@ -1857,8 +1838,7 @@ where } } -impl - CancellableFutureWithReason, ChildWorkflowExecutionError>> +impl CancellableFutureWithReason, ChildWorkflowStartError>> for ChildWorkflowStartFut where F: CancellableFutureWithReason> + Unpin, @@ -1878,7 +1858,10 @@ enum SignalChildFut { Errored { error: Option, }, - Running(F), + Running { + inner: F, + data_converter: DataConverter, + }, Terminated, } @@ -1902,12 +1885,17 @@ where SignalChildFut::Errored { error } => { Poll::Ready(Err(error.take().expect("polled after completion"))) } - SignalChildFut::Running(inner) => match Pin::new(inner).poll(cx) { + SignalChildFut::Running { + inner, + data_converter, + } => match Pin::new(inner).poll(cx) { Poll::Pending => Poll::Pending, Poll::Ready(Ok(_)) => Poll::Ready(Ok(())), - Poll::Ready(Err(failure)) => { - Poll::Ready(Err(ChildWorkflowSignalError::Failed(Box::new(failure)))) - } + Poll::Ready(Err(failure)) => Poll::Ready(Err(data_converter.to_error( + &SerializationContextData::Workflow, + failure, + ChildWorkflowSignalDecodeHint, + )?)), }, SignalChildFut::Terminated => panic!("polled after termination"), }; @@ -1932,7 +1920,7 @@ where F: CancellableFuture + Unpin, { fn cancel(&self) { - if let SignalChildFut::Running(inner) = self { + if let SignalChildFut::Running { inner, .. } = self { inner.cancel() } } @@ -1949,7 +1937,7 @@ where ) -> impl CancellableFutureWithReason> { ChildWorkflowFut::Running { inner: self.common.result_future, - payload_converter: self.common.payload_converter, + data_converter: self.common.data_converter, _phantom: PhantomData, } } @@ -1971,11 +1959,12 @@ where signal: S, input: S::Input, ) -> impl CancellableFuture> + 'static { + let payload_converter = self.common.data_converter.payload_converter(); let ctx = SerializationContext { data: &SerializationContextData::Workflow, - converter: &self.common.payload_converter, + converter: payload_converter, }; - let payloads = match self.common.payload_converter.to_payloads(&ctx, &input) { + let payloads = match payload_converter.to_payloads(&ctx, &input) { Ok(p) => p, Err(e) => { return SignalChildFut::eager(e.into()); @@ -1983,7 +1972,10 @@ where }; let signal = Signal::new(S::name(&signal), payloads); let target = sig_we::Target::ChildWorkflowId(self.common.workflow_id.clone()); - SignalChildFut::Running(self.common.base_ctx.clone().send_signal_wf(target, signal)) + SignalChildFut::Running { + inner: self.common.base_ctx.clone().send_signal_wf(target, signal), + data_converter: self.common.data_converter.clone(), + } } } @@ -2017,16 +2009,12 @@ impl ExternalWorkflowHandle { signal: S, input: S::Input, ) -> impl CancellableFuture + 'static { + let payload_converter = self.base_ctx.inner.data_converter.payload_converter(); let ctx = SerializationContext { data: &SerializationContextData::Workflow, - converter: &self.base_ctx.inner.payload_converter, + converter: payload_converter, }; - let payloads = match self - .base_ctx - .inner - .payload_converter - .to_payloads(&ctx, &input) - { + let payloads = match payload_converter.to_payloads(&ctx, &input) { Ok(p) => p, Err(e) => { return SignalExternalFut::SerializationError(Some(e)); @@ -2196,7 +2184,7 @@ mod tests { "run-id".to_string(), init, cancelled_rx, - PayloadConverter::default(), + DataConverter::default(), ); WorkflowContext::from_base(base, Rc::new(RefCell::new(TestWorkflow))) } @@ -2350,7 +2338,7 @@ mod tests { "run-id".to_string(), init, cancelled_rx, - PayloadConverter::default(), + DataConverter::default(), ); let ctx = WorkflowContext::from_base(base, Rc::new(RefCell::new(FailingWorkflow))); diff --git a/crates/sdk/src/workflow_future.rs b/crates/sdk/src/workflow_future.rs index 57cbe45cf..9d2f46392 100644 --- a/crates/sdk/src/workflow_future.rs +++ b/crates/sdk/src/workflow_future.rs @@ -1,6 +1,6 @@ use crate::{ - BaseWorkflowContext, CancellableID, RustWfCmd, TimerResult, UnblockEvent, WorkflowResult, - WorkflowTermination, panic_formatter, + BaseWorkflowContext, CancellableID, OutgoingError, OutgoingWorkflowError, RustWfCmd, + TimerResult, UnblockEvent, WorkflowResult, WorkflowTermination, panic_formatter, workflow_executor::{SdkWakeGuard, WakeTracker}, workflows::{DispatchData, DynWorkflowExecution, WorkflowExecutionFactory}, }; @@ -17,7 +17,8 @@ use std::{ }; use temporalio_common::{ data_converters::{ - GenericPayloadConverter, PayloadConverter, SerializationContext, SerializationContextData, + DataConverter, GenericPayloadConverter, PayloadConverter, SerializationContext, + SerializationContextData, }, protos::{ coresdk::{ @@ -69,7 +70,7 @@ impl WorkflowFunction { run_id: String, init_workflow_job: InitializeWorkflow, outgoing_completions: UnboundedSender, - payload_converter: PayloadConverter, + data_converter: DataConverter, detect_nondeterministic: bool, ) -> Result< ( @@ -85,6 +86,7 @@ impl WorkflowFunction { "otel.kind" = "server" ); + let payload_converter = data_converter.payload_converter().clone(); let input = init_workflow_job.arguments.clone(); let (base_ctx, cmd_receiver) = BaseWorkflowContext::new( namespace, @@ -92,11 +94,11 @@ impl WorkflowFunction { run_id, init_workflow_job, cancel_rx, - payload_converter.clone(), + data_converter.clone(), ); // Create the workflow execution using the factory - let execution = (self.factory)(input, payload_converter.clone(), base_ctx.clone()) + let execution = (self.factory)(input, payload_converter, base_ctx.clone()) .context("Failed to create workflow execution")?; let wake_tracking = if detect_nondeterministic { @@ -116,7 +118,7 @@ impl WorkflowFunction { incoming_activations, command_status: Default::default(), cancel_sender: cancel_tx, - payload_converter, + data_converter, update_futures: Default::default(), signal_futures: Default::default(), wake_tracking, @@ -147,8 +149,8 @@ pub(crate) struct WorkflowFuture { cancel_sender: watch::Sender>, /// Base workflow context for sending commands base_ctx: BaseWorkflowContext, - /// Payload converter for serialization/deserialization - payload_converter: PayloadConverter, + /// Data converter for workflow failure conversion and payload serialization. + data_converter: DataConverter, /// Stores in-progress update futures update_futures: Vec<( String, @@ -162,6 +164,23 @@ pub(crate) struct WorkflowFuture { } impl WorkflowFuture { + fn workflow_message_to_failure(&self, message: String) -> Failure { + self.workflow_error_to_failure(anyhow!(message).into()) + } + + fn workflow_error_to_failure(&self, error: crate::workflows::WorkflowError) -> Failure { + let outgoing = match error { + crate::workflows::WorkflowError::PayloadConversion(err) => { + OutgoingWorkflowError::from(err) + } + crate::workflows::WorkflowError::Execution(err) => err.into(), + }; + self.data_converter.to_failure( + &SerializationContextData::Workflow, + OutgoingError::Workflow(outgoing), + ) + } + fn unblock(&mut self, event: UnblockEvent) -> Result<(), Error> { let cmd_id = match event { UnblockEvent::Timer(seq, _) => CommandID::Timer(seq), @@ -262,7 +281,7 @@ impl WorkflowFuture { payloads: q.arguments, }, headers: q.headers, - converter: &self.payload_converter, + converter: self.data_converter.payload_converter(), }; let dispatch_result = if query_type == "__temporal_workflow_metadata" { @@ -306,14 +325,12 @@ impl WorkflowFuture { response: Some(payload), }), // TODO [rust-sdk-branch]: Return list of known queries in error - None => query_result::Variant::Failed(Failure { - message: format!("No query handler for '{}'", query_type), - ..Default::default() - }), - Some(Err(e)) => query_result::Variant::Failed(Failure { - message: e.to_string(), - ..Default::default() - }), + None => query_result::Variant::Failed(self.workflow_message_to_failure( + format!("No query handler for '{}'", query_type), + )), + Some(Err(e)) => { + query_result::Variant::Failed(self.workflow_error_to_failure(e)) + } }; outgoing_cmds.push( @@ -339,7 +356,7 @@ impl WorkflowFuture { payloads: sig.input, }, headers: sig.headers, - converter: &self.payload_converter, + converter: self.data_converter.payload_converter(), }; let dispatch_result = match panic::catch_unwind(AssertUnwindSafe(|| { @@ -371,7 +388,7 @@ impl WorkflowFuture { let data = DispatchData { payloads: Payloads { payloads: u.input }, headers: u.headers, - converter: &self.payload_converter, + converter: self.data_converter.payload_converter(), }; let trait_val_result = if u.run_validator { @@ -408,7 +425,9 @@ impl WorkflowFuture { outgoing_cmds.push( update_response( protocol_instance_id.clone(), - update_response::Response::Rejected(anyhow!(e).into()), + update_response::Response::Rejected( + self.workflow_error_to_failure(e), + ), ) .into(), ); @@ -473,9 +492,10 @@ impl Future for WorkflowFuture { Poll::Ready(a) => match a { Some(act) => act, None => { - return Poll::Ready(Err(WorkflowTermination::failed(anyhow!( + return Poll::Ready(Err(anyhow!( "Workflow future's activation channel was lost!" - )))); + ) + .into())); } }, Poll::Pending => return Poll::Pending, @@ -608,7 +628,9 @@ impl WorkflowFuture { instance_id, match v { Ok(v) => update_response::Response::Completed(v), - Err(e) => update_response::Response::Rejected(e.into()), + Err(e) => update_response::Response::Rejected( + self.workflow_error_to_failure(e), + ), }, ) .into(), @@ -652,10 +674,14 @@ impl WorkflowFuture { self.outgoing_completions .send(WorkflowActivationCompletion::fail( run_id, - Failure { - message: errmsg, - ..Default::default() - }, + self.data_converter.to_failure( + &SerializationContextData::Workflow, + OutgoingError::Workflow(OutgoingWorkflowError::Application( + Box::new(crate::ApplicationFailure::non_retryable(anyhow!( + "{errmsg}" + ))), + )), + ), None, )) .expect("Completion channel intact"); @@ -790,10 +816,10 @@ impl WorkflowFuture { } WorkflowTermination::Failed(e) => { workflow_command::Variant::FailWorkflowExecution(FailWorkflowExecution { - failure: Some(Failure { - message: format!("Workflow execution error: {e}"), - ..Default::default() - }), + failure: Some(self.data_converter.to_failure( + &SerializationContextData::Workflow, + OutgoingError::Workflow(e), + )), }) } },