fix(spider-storage): Reserve UNAVAILABLE gRPC status code for network connection errors.

components/spider-execution-manager/src/client/grpc/storage.rs

@@ -135,15 +135,17 @@ impl StorageClient for GrpcStorageClient {
 ///
 /// The [`StorageResponseError`] for `status`'s code:
 ///
-/// * [`StorageResponseError::StaleSession`] for `UNAVAILABLE`.
+/// * [`StorageResponseError::StaleSession`] for `NOT_FOUND`.
 /// * [`StorageResponseError::CacheStale`] for `FAILED_PRECONDITION`.
 /// * [`StorageResponseError::InvalidInput`] for `INVALID_ARGUMENT`.
+/// * [`StorageResponseError::Transport`] for `UNAVAILABLE` (a lost or unestablished connection).
 /// * [`StorageResponseError::Server`] for any other code.
 fn status_to_error(status: &Status) -> StorageResponseError {
     match status.code() {
-        Code::Unavailable => StorageResponseError::StaleSession(status.message().to_owned()),
+        Code::NotFound => StorageResponseError::StaleSession(status.message().to_owned()),
         Code::FailedPrecondition => StorageResponseError::CacheStale(status.message().to_owned()),
         Code::InvalidArgument => StorageResponseError::InvalidInput(status.message().to_owned()),
+        Code::Unavailable => to_transport_error(status.message()),
         _ => StorageResponseError::Server(status.message().to_owned()),
     }
 }
@@ -162,17 +164,29 @@ mod tests {
     use super::*;
 
     #[test]
-    fn status_maps_unavailable_to_stale_session() {
-        match status_to_error(&Status::unavailable("storage session is 9")) {
+    fn status_maps_not_found_to_stale_session() {
+        match status_to_error(&Status::not_found("storage session is 9")) {
             StorageResponseError::StaleSession(message) => assert!(message.contains('9')),
             error => panic!("unexpected error: {error:?}"),
         }
     }
 
+    #[test]
+    fn status_maps_unavailable_to_transport() {
+        const MESSAGE: &str = "connection lost";
+        match status_to_error(&Status::unavailable(MESSAGE)) {
+            StorageResponseError::Transport(message) => {
+                assert!(message.contains(MESSAGE));
+            }
+            error => panic!("unexpected error: {error:?}"),
+        }
+    }
+
     #[test]
     fn status_maps_invalid_argument_to_invalid_input() {
-        match status_to_error(&Status::invalid_argument("bad task id")) {
-            StorageResponseError::InvalidInput(message) => assert!(message.contains("bad task id")),
+        const MESSAGE: &str = "bad task id";
+        match status_to_error(&Status::invalid_argument(MESSAGE)) {
+            StorageResponseError::InvalidInput(message) => assert!(message.contains(MESSAGE)),
             error => panic!("unexpected error: {error:?}"),
         }
     }

components/spider-storage/src/grpc.rs

@@ -75,12 +75,13 @@ impl<
     ///
     /// The [`Status`] to send to the client:
     ///
-    /// * `UNAVAILABLE` for a fatal cache-internal error (the service will restart).
+    /// * `INTERNAL` for:
+    ///   * A fatal cache-internal error (the service will restart).
+    ///   * Any other (database or otherwise unexpected) error.
     /// * `UNAUTHENTICATED` for an unknown or unauthorized resource group.
     /// * `NOT_FOUND` for a missing job.
     /// * `FAILED_PRECONDITION` for operations on an invalid job state.
     /// * `INVALID_ARGUMENT` for a malformed task graph, inputs, or request.
-    /// * `INTERNAL` for any other (database or otherwise unexpected) error.
     pub fn job_orchestration_service_error_handler(
         &self,
         error: StorageServerError,
@@ -96,7 +97,7 @@ impl<
                     "Internal error in the cache layer. Cancelling service."
                 );
                 self.cancellation_token.cancel();
-                Status::unavailable("storage service unavailable")
+                Status::internal("storage service unavailable")
             }
 
             StorageServerError::Db(db_error) => match &db_error {
@@ -182,7 +183,7 @@ impl<
     /// * `INTERNAL` for:
     ///   * A fatal cache-internal error (the service will restart).
     ///   * Any other unexpected error (the service will restart).
-    /// * `UNAVAILABLE` for a request issued from a stale session.
+    /// * `NOT_FOUND` for a request issued from a stale session.
     /// * `FAILED_PRECONDITION` for a request issued against a stale cache state.
     /// * `INVALID_ARGUMENT` for malformed inputs or a malformed request.
     pub fn task_instance_management_service_error_handler(
@@ -210,7 +211,7 @@ impl<
                     tag,
                     "The request was issued from a stale session."
                 );
-                Status::unavailable(format!(
+                Status::not_found(format!(
                     "stale session; current storage session is {storage_session}"
                 ))
             }