fix(sourcemap): Initial implementation for sourcemap processing (#992)

crates/symbolicator-service/Cargo.toml

@@ -37,7 +37,8 @@ sentry = { version = "0.29.2", features = ["tracing"] }
 serde = { version = "1.0.137", features = ["derive", "rc"] }
 serde_json = "1.0.81"
 serde_yaml = "0.9.14"
-symbolic = { version = "11.0.0", features = ["cfi", "common-serde", "debuginfo", "demangle", "symcache", "il2cpp", "ppdb"] }
+symbolic = { version = "11.0.0", features = ["cfi", "common-serde", "debuginfo", "demangle", "sourcemapcache", "symcache", "il2cpp", "ppdb"] }
+sourcemap = "6.2.1"
 symbolicator-sources = { path = "../symbolicator-sources" }
 tempfile = "3.2.0"
 thiserror = "1.0.31"

crates/symbolicator-service/src/services/download/mod.rs

@@ -20,11 +20,13 @@ pub use symbolicator_sources::{
     SourceFilters, SourceLocation,
 };
 use symbolicator_sources::{
-    FilesystemRemoteFile, GcsRemoteFile, HttpRemoteFile, S3RemoteFile, SourceLocationIter,
+    FilesystemRemoteFile, GcsRemoteFile, HttpRemoteFile, S3RemoteFile, SentrySourceConfig,
+    SourceLocationIter,
 };
 
 use crate::caching::{CacheEntry, CacheError};
 use crate::config::{CacheConfigs, Config, InMemoryCacheConfig};
+use crate::services::download::sentry::SearchArtifactResult;
 use crate::utils::futures::{m, measure, CancelOnDrop};
 use crate::utils::gcs::GcsError;
 use crate::utils::sentry::ConfigureScope;
@@ -33,7 +35,7 @@ mod filesystem;
 mod gcs;
 mod http;
 mod s3;
-mod sentry;
+pub mod sentry;
 
 impl ConfigureScope for RemoteFile {
     fn to_scope(&self, scope: &mut ::sentry::Scope) {
@@ -267,6 +269,26 @@ impl DownloadService {
         }
         remote_files
     }
+
+    pub async fn list_artifacts(
+        &self,
+        source: Arc<SentrySourceConfig>,
+    ) -> Vec<SearchArtifactResult> {
+        let mut remote_artifacts = vec![];
+        let job = self.sentry.list_artifacts(source.clone());
+        let timeout = Duration::from_secs(30);
+        let job = tokio::time::timeout(timeout, job);
+        let job = measure("service.download.list_artifacts", m::timed_result, job);
+        let sentry_files = job.await.map_err(|_| CacheError::Timeout(timeout));
+        match sentry_files {
+            Ok(Ok(files)) => remote_artifacts.extend(files),
+            Ok(Err(error)) | Err(error) => {
+                let error: &dyn std::error::Error = &error;
+                tracing::error!(error, "Failed to fetch artifacts list");
+            }
+        }
+        remote_artifacts
+    }
 }
 
 /// Try to run a future up to 3 times with 20 millisecond delays on failure.

crates/symbolicator-service/src/services/download/sentry.rs

@@ -2,27 +2,39 @@
 //!
 //! This allows to fetch files which were directly uploaded to Sentry itself.
 
+use std::collections::BTreeMap;
 use std::fmt;
 use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 
 use sentry::SentryFutureExt;
+use serde::de::DeserializeOwned;
+use serde::Deserialize;
 use url::Url;
 
 use symbolicator_sources::{
-    ObjectId, RemoteFile, SentryFileId, SentryRemoteFile, SentrySourceConfig,
+    ObjectId, RemoteFile, SentryFileId, SentryFileType, SentryRemoteFile, SentrySourceConfig,
 };
 
 use super::{FileType, USER_AGENT};
 use crate::caching::{CacheEntry, CacheError};
 use crate::config::Config;
 use crate::utils::futures::CancelOnDrop;
 
-#[derive(Clone, Debug, serde::Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
 struct SearchResult {
-    id: SentryFileId,
-    // TODO: Add more fields
+    pub id: SentryFileId,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct SearchArtifactResult {
+    pub id: SentryFileId,
+    pub name: String,
+    pub sha1: String,
+    pub dist: Option<String>,
+    #[serde(default)]
+    pub headers: BTreeMap<String, String>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
@@ -66,10 +78,13 @@ impl SentryDownloader {
     }
 
     /// Make a request to sentry, parse the result as a JSON SearchResult list.
-    async fn fetch_sentry_json(
+    async fn fetch_sentry_json<T>(
         client: &reqwest::Client,
         query: &SearchQuery,
-    ) -> CacheEntry<Vec<SearchResult>> {
+    ) -> CacheEntry<Vec<T>>
+    where
+        T: DeserializeOwned,
+    {
         let mut request = client
             .get(query.index_url.clone())
             .bearer_auth(&query.token)
@@ -174,18 +189,52 @@ impl SentryDownloader {
         let search = self.cached_sentry_search(query).await?;
         let file_ids = search
             .into_iter()
-            .map(|search_result| SentryRemoteFile::new(source.clone(), search_result.id).into())
+            .map(|search_result| {
+                SentryRemoteFile::new(source.clone(), search_result.id, SentryFileType::DebugFile)
+                    .into()
+            })
             .collect();
 
         Ok(file_ids)
     }
 
+    pub async fn list_artifacts(
+        &self,
+        source: Arc<SentrySourceConfig>,
+    ) -> CacheEntry<Vec<SearchArtifactResult>> {
+        let query = SearchQuery {
+            index_url: source.url.clone(),
+            token: source.token.clone(),
+        };
+
+        tracing::debug!(
+            "Fetching list of Sentry artifacts from {}",
+            &query.index_url
+        );
+
+        let entries = {
+            let client = self.client.clone();
+            let query = query.clone();
+            let future =
+                async move { super::retry(|| Self::fetch_sentry_json(&client, &query)).await };
+
+            let future =
+                CancelOnDrop::new(self.runtime.spawn(future.bind_hub(sentry::Hub::current())));
+
+            future.await.map_err(|_| CacheError::InternalError)??
+        };
+
+        Ok(entries)
+    }
+
     /// Downloads a source hosted on Sentry.
     pub async fn download_source(
         &self,
         file_source: SentryRemoteFile,
         destination: &Path,
     ) -> CacheEntry {
+        tracing::debug!("Fetching Sentry artifact from {}", file_source.url());
+
         let request = self
             .client
             .get(file_source.url())
@@ -217,7 +266,11 @@ mod tests {
             url: Url::parse("https://example.net/endpoint/").unwrap(),
             token: "token".into(),
         };
-        let file_source = SentryRemoteFile::new(Arc::new(source), SentryFileId("abc123".into()));
+        let file_source = SentryRemoteFile::new(
+            Arc::new(source),
+            SentryFileId("abc123".into()),
+            SentryFileType::DebugFile,
+        );
         let url = file_source.url();
         assert_eq!(url.as_str(), "https://example.net/endpoint/?id=abc123");
     }
@@ -229,7 +282,11 @@ mod tests {
             url: Url::parse("https://example.net/endpoint/").unwrap(),
             token: "token".into(),
         };
-        let file_source = SentryRemoteFile::new(Arc::new(source), SentryFileId("abc123".into()));
+        let file_source = SentryRemoteFile::new(
+            Arc::new(source),
+            SentryFileId("abc123".into()),
+            SentryFileType::DebugFile,
+        );
         let uri = file_source.uri();
         assert_eq!(
             uri,

crates/symbolicator-service/src/services/mod.rs

@@ -23,6 +23,7 @@ pub mod il2cpp;
 mod minidump;
 pub mod objects;
 pub mod ppdb_caches;
+pub mod sourcemap;
 pub mod symbolication;
 pub mod symcaches;
 
@@ -32,6 +33,7 @@ use self::download::DownloadService;
 use self::il2cpp::Il2cppService;
 use self::objects::ObjectsActor;
 use self::ppdb_caches::PortablePdbCacheActor;
+use self::sourcemap::SourceMapService;
 use self::symbolication::SymbolicationActor;
 use self::symcaches::SymCacheActor;
 pub use fetch_file::fetch_file;
@@ -58,7 +60,7 @@ pub fn create_service(
 
     let bitcode = BitcodeService::new(caches.auxdifs, shared_cache.clone(), downloader.clone());
 
-    let il2cpp = Il2cppService::new(caches.il2cpp, shared_cache.clone(), downloader);
+    let il2cpp = Il2cppService::new(caches.il2cpp, shared_cache.clone(), downloader.clone());
 
     let symcaches = SymCacheActor::new(
         caches.symcaches,
@@ -72,12 +74,15 @@ pub fn create_service(
 
     let ppdb_caches = PortablePdbCacheActor::new(caches.ppdb_caches, shared_cache, objects.clone());
 
+    let sourcemaps = SourceMapService::new(downloader);
+
     let symbolication = SymbolicationActor::new(
         objects.clone(),
         symcaches,
         cficaches,
         ppdb_caches,
         caches.diagnostics,
+        sourcemaps,
     );
 
     Ok((symbolication, objects))

crates/symbolicator-service/src/services/sourcemap.rs

@@ -0,0 +1,51 @@
+//! Service for retrieving SourceMap artifacts.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use symbolicator_sources::{SentryFileId, SentryFileType, SentryRemoteFile, SentrySourceConfig};
+use tempfile::NamedTempFile;
+
+use crate::services::download::sentry::SearchArtifactResult;
+use crate::services::download::DownloadService;
+
+use super::fetch_file;
+
+#[derive(Debug, Clone)]
+pub struct SourceMapService {
+    download_svc: Arc<DownloadService>,
+}
+
+impl SourceMapService {
+    pub fn new(download_svc: Arc<DownloadService>) -> Self {
+        Self { download_svc }
+    }
+
+    pub async fn list_artifacts(
+        &self,
+        source: Arc<SentrySourceConfig>,
+    ) -> HashMap<String, SearchArtifactResult> {
+        self.download_svc
+            .list_artifacts(source)
+            .await
+            .into_iter()
+            .map(|artifact| (artifact.name.clone(), artifact))
+            .collect()
+    }
+
+    pub async fn fetch_artifact(
+        &self,
+        source: Arc<SentrySourceConfig>,
+        file_id: SentryFileId,
+    ) -> Option<NamedTempFile> {
+        let mut temp_file = NamedTempFile::new().unwrap();
+        fetch_file(
+            self.download_svc.clone(),
+            SentryRemoteFile::new(source, file_id, SentryFileType::ReleaseArtifact).into(),
+            &mut temp_file,
+        )
+        .await
+        .map(|_| temp_file)
+        .ok()
+    }
+}

crates/symbolicator-service/src/services/symbolication/mod.rs

@@ -4,19 +4,19 @@ use symbolic::common::{split_path, DebugId, InstructionInfo, Language, Name};
 use symbolic::demangle::{Demangle, DemangleOptions};
 use symbolic::ppdb::PortablePdbCache;
 use symbolic::symcache::SymCache;
-use symbolicator_sources::ObjectId;
+use symbolicator_sources::{ObjectId, SentrySourceConfig};
 use symbolicator_sources::{ObjectType, SourceConfig};
 
 use crate::caching::{Cache, CacheError};
 use crate::services::cficaches::CfiCacheActor;
 use crate::services::objects::ObjectsActor;
 use crate::services::ppdb_caches::PortablePdbCacheActor;
+use crate::services::sourcemap::SourceMapService;
 use crate::services::symcaches::SymCacheActor;
-use crate::types::RawObjectInfo;
 use crate::types::{
     CompleteObjectInfo, CompleteStacktrace, CompletedSymbolicationResponse, FrameStatus,
-    FrameTrust, ObjectFileStatus, RawFrame, RawStacktrace, Registers, Scope, Signal,
-    SymbolicatedFrame,
+    FrameTrust, JsProcessingRawStacktrace, ObjectFileStatus, RawFrame, RawObjectInfo,
+    RawStacktrace, Registers, Scope, Signal, SymbolicatedFrame,
 };
 use crate::utils::hex::HexValue;
 
@@ -25,6 +25,7 @@ use module_lookup::{CacheFileEntry, CacheLookupResult, ModuleLookup};
 mod apple;
 mod module_lookup;
 mod process_minidump;
+pub mod sourcemap;
 
 fn object_id_from_object_info(object_info: &RawObjectInfo) -> ObjectId {
     ObjectId {
@@ -98,6 +99,7 @@ pub struct SymbolicationActor {
     cficaches: CfiCacheActor,
     ppdb_caches: PortablePdbCacheActor,
     diagnostics_cache: Cache,
+    sourcemaps: SourceMapService,
 }
 
 impl SymbolicationActor {
@@ -107,13 +109,15 @@ impl SymbolicationActor {
         cficaches: CfiCacheActor,
         ppdb_caches: PortablePdbCacheActor,
         diagnostics_cache: Cache,
+        sourcemaps: SourceMapService,
     ) -> Self {
         SymbolicationActor {
             objects,
             symcaches,
             cficaches,
             ppdb_caches,
             diagnostics_cache,
+            sourcemaps,
         }
     }
 
@@ -221,6 +225,13 @@ pub struct SymbolicateStacktraces {
     pub modules: Vec<CompleteObjectInfo>,
 }
 
+#[derive(Debug, Clone)]
+pub struct JsProcessingSymbolicateStacktraces {
+    pub source: Arc<SentrySourceConfig>,
+    pub stacktraces: Vec<JsProcessingRawStacktrace>,
+    pub dist: Option<String>,
+}
+
 fn symbolicate_frame(
     caches: &ModuleLookup,
     registers: &Registers,