Add a basic proguard workspace crate (#1371)

For now symbolicator-proguard includes only a ProguardService that can download proguard files from Sentry sources and turn them into ProguardMapper structs.

-----------------------------
Co-authored-by: Yagiz Nizipli <[email protected]>
Co-authored-by: Sebastian Zivota <[email protected]>

crates/symbolicator-proguard/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "symbolicator-proguard"
+publish = false
+version = "24.1.2"
+authors = ["Sentry <[email protected]>"]
+edition = "2021"
+license = "MIT"
+
+[dependencies]
+futures = "0.3.12"
+proguard = "5.4.0"
+serde = { version = "1.0.137", features = ["derive", "rc"] }
+serde_json = "1.0.81"
+symbolic = "12.8.0"
+symbolicator-service = { path = "../symbolicator-service" }
+symbolicator-sources = { path = "../symbolicator-sources" }
+tempfile = "3.10.0"
+
+[dev-dependencies]
+symbolicator-test = { path = "../symbolicator-test" }
+tokio = { version = "1.24.2", features = ["rt", "macros", "fs"] }

crates/symbolicator-proguard/src/interface.rs

@@ -0,0 +1 @@
+

crates/symbolicator-proguard/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod interface;
+mod service;
+
+pub use service::ProguardService;

crates/symbolicator-proguard/src/service.rs

@@ -0,0 +1,150 @@
+use std::sync::Arc;
+
+use futures::future::BoxFuture;
+use symbolic::common::{AsSelf, ByteView, DebugId, SelfCell};
+use symbolicator_service::caches::versions::PROGUARD_CACHE_VERSIONS;
+use symbolicator_service::caching::{
+    CacheEntry, CacheError, CacheItemRequest, CacheKey, CacheVersions, Cacher,
+};
+use symbolicator_service::download::{fetch_file, DownloadService};
+use symbolicator_service::services::SharedServices;
+use symbolicator_service::types::Scope;
+use symbolicator_sources::{FileType, ObjectId, RemoteFile, SourceConfig};
+use tempfile::NamedTempFile;
+
+#[derive(Debug, Clone)]
+pub struct ProguardService {
+    pub(crate) download_svc: Arc<DownloadService>,
+    pub(crate) cache: Arc<Cacher<FetchProguard>>,
+}
+
+impl ProguardService {
+    pub fn new(services: &SharedServices) -> Self {
+        let caches = &services.caches;
+        let shared_cache = services.shared_cache.clone();
+        let download_svc = services.download_svc.clone();
+
+        let cache = Arc::new(Cacher::new(caches.proguard.clone(), shared_cache));
+
+        Self {
+            download_svc,
+            cache,
+        }
+    }
+
+    async fn find_proguard_file(
+        &self,
+        sources: &[SourceConfig],
+        identifier: &ObjectId,
+    ) -> Option<RemoteFile> {
+        let file_ids = self
+            .download_svc
+            .list_files(sources, &[FileType::Proguard], identifier)
+            .await;
+
+        file_ids.into_iter().next()
+    }
+
+    /// Retrieves the given [`RemoteFile`] from cache, or fetches it and persists it according
+    /// to the provided [`Scope`].
+    /// It is possible to avoid using the shared cache using the `use_shared_cache` parameter.
+    pub async fn fetch_file(&self, scope: &Scope, file: RemoteFile) -> CacheEntry<ProguardMapper> {
+        let cache_key = CacheKey::from_scoped_file(scope, &file);
+
+        let request = FetchProguard {
+            file,
+            download_svc: Arc::clone(&self.download_svc),
+        };
+
+        self.cache.compute_memoized(request, cache_key).await
+    }
+
+    pub async fn download_proguard_file(
+        &self,
+        sources: &[SourceConfig],
+        scope: &Scope,
+        debug_id: DebugId,
+    ) -> CacheEntry<ProguardMapper> {
+        let identifier = ObjectId {
+            debug_id: Some(debug_id),
+            ..Default::default()
+        };
+
+        let remote_file = self
+            .find_proguard_file(sources, &identifier)
+            .await
+            .ok_or(CacheError::NotFound)?;
+
+        self.fetch_file(scope, remote_file).await
+    }
+}
+
+struct ProguardInner<'a> {
+    // TODO: actually use it
+    #[allow(unused)]
+    mapping: proguard::ProguardMapping<'a>,
+    // TODO: actually use it
+    #[allow(unused)]
+    mapper: proguard::ProguardMapper<'a>,
+}
+
+impl<'slf, 'a: 'slf> AsSelf<'slf> for ProguardInner<'a> {
+    type Ref = ProguardInner<'slf>;
+
+    fn as_self(&'slf self) -> &Self::Ref {
+        self
+    }
+}
+
+#[derive(Clone)]
+pub struct ProguardMapper {
+    // TODO: actually use it
+    #[allow(unused)]
+    inner: Arc<SelfCell<ByteView<'static>, ProguardInner<'static>>>,
+}
+
+#[derive(Clone, Debug)]
+pub struct FetchProguard {
+    file: RemoteFile,
+    download_svc: Arc<DownloadService>,
+}
+
+impl CacheItemRequest for FetchProguard {
+    type Item = ProguardMapper;
+
+    const VERSIONS: CacheVersions = PROGUARD_CACHE_VERSIONS;
+
+    fn compute<'a>(&'a self, temp_file: &'a mut NamedTempFile) -> BoxFuture<'a, CacheEntry> {
+        let fut = async {
+            fetch_file(self.download_svc.clone(), self.file.clone(), temp_file).await?;
+
+            let view = ByteView::map_file_ref(temp_file.as_file())?;
+
+            let mapping = proguard::ProguardMapping::new(&view);
+            if mapping.is_valid() {
+                Ok(())
+            } else {
+                Err(CacheError::Malformed(
+                    "The file is not a valid ProGuard file".into(),
+                ))
+            }
+        };
+        Box::pin(fut)
+    }
+
+    fn load(&self, byteview: ByteView<'static>) -> CacheEntry<Self::Item> {
+        let inner = SelfCell::new(byteview, |data| {
+            let mapping = proguard::ProguardMapping::new(unsafe { &*data });
+            let mapper = proguard::ProguardMapper::new(mapping.clone());
+            ProguardInner { mapping, mapper }
+        });
+
+        Ok(ProguardMapper {
+            inner: Arc::new(inner),
+        })
+    }
+
+    fn use_shared_cache(&self) -> bool {
+        false
+    }
+}

crates/symbolicator-proguard/tests/integration/main.rs

@@ -0,0 +1,4 @@
+mod proguard;
+mod utils;
+
+pub use utils::*;

crates/symbolicator-proguard/tests/integration/proguard.rs

@@ -0,0 +1,54 @@
+use std::sync::Arc;
+use std::{collections::HashMap, str::FromStr};
+
+use serde_json::json;
+use symbolic::common::DebugId;
+use symbolicator_service::types::Scope;
+use symbolicator_sources::{SentrySourceConfig, SourceConfig};
+
+use crate::setup_service;
+
+fn proguard_server<L>(
+    fixtures_dir: &str,
+    lookup: L,
+) -> (symbolicator_test::Server, SentrySourceConfig)
+where
+    L: Fn(&str, &HashMap<String, String>) -> serde_json::Value + Clone + Send + 'static,
+{
+    let fixtures_dir = symbolicator_test::fixture(format!("proguard/{fixtures_dir}"));
+    symbolicator_test::sentry_server(fixtures_dir, lookup)
+}
+
+#[tokio::test]
+async fn test_download_proguard_file() {
+    symbolicator_test::setup();
+    let (symbolication, _cache_dir) = setup_service(|_| ());
+    let (_srv, source) = proguard_server("01", |_url, _query| {
+        json!([{
+            "id":"proguard.txt",
+            "uuid":"246fb328-fc4e-406a-87ff-fc35f6149d8f",
+            "debugId":"246fb328-fc4e-406a-87ff-fc35f6149d8f",
+            "codeId":null,
+            "cpuName":"any",
+            "objectName":"proguard-mapping",
+            "symbolType":"proguard",
+            "headers": {
+                "Content-Type":"text/x-proguard+plain"
+            },
+            "size":3619,
+            "sha1":"deba83e73fd18210a830db372a0e0a2f2293a989",
+            "dateCreated":"2024-02-14T10:49:38.770116Z",
+            "data":{
+                "features":["mapping"]
+            }
+        }])
+    });
+
+    let source = SourceConfig::Sentry(Arc::new(source));
+    let debug_id = DebugId::from_str("246fb328-fc4e-406a-87ff-fc35f6149d8f").unwrap();
+
+    assert!(symbolication
+        .download_proguard_file(&[source], &Scope::Global, debug_id)
+        .await
+        .is_ok());
+}

crates/symbolicator-proguard/tests/integration/utils.rs

@@ -0,0 +1,35 @@
+use symbolicator_proguard::ProguardService;
+use symbolicator_service::config::Config;
+use symbolicator_service::services::SharedServices;
+use symbolicator_test as test;
+
+pub use test::{assert_snapshot, fixture, read_fixture, source_config, symbol_server, Server};
+
+/// Setup tests and create a test service.
+///
+/// This function returns a tuple containing the service to test, and a temporary cache
+/// directory. The directory is cleaned up when the [`TempDir`] instance is dropped. Keep it as
+/// guard until the test has finished.
+///
+/// The service is configured with `connect_to_reserved_ips = True`. This allows to use a local
+/// symbol server to test object file downloads.
+/// The `update_config` closure can modify any default configuration if needed before the server is
+/// started.
+pub fn setup_service(update_config: impl FnOnce(&mut Config)) -> (ProguardService, test::TempDir) {
+    test::setup();
+
+    let cache_dir = test::tempdir();
+
+    let mut config = Config {
+        cache_dir: Some(cache_dir.path().to_owned()),
+        connect_to_reserved_ips: true,
+        ..Default::default()
+    };
+    update_config(&mut config);
+
+    let handle = tokio::runtime::Handle::current();
+    let shared_services = SharedServices::new(config, handle).unwrap();
+    let proguard = ProguardService::new(&shared_services);
+
+    (proguard, cache_dir)
+}

crates/symbolicator-service/src/caches/versions.rs

@@ -138,3 +138,11 @@ pub const BUNDLE_INDEX_CACHE_VERSIONS: CacheVersions = CacheVersions {
     current: 1,
     fallbacks: &[],
 };
+
+/// Proguard Cache, with the following versions:
+///
+/// - `1`: Initial version.
+pub const PROGUARD_CACHE_VERSIONS: CacheVersions = CacheVersions {
+    current: 1,
+    fallbacks: &[],
+};

crates/symbolicator-service/src/caching/cleanup.rs

@@ -47,6 +47,7 @@ impl Caches {
             sourcefiles,
             bundle_index,
             diagnostics,
+            proguard,
         } = &self;
 
         // We want to clean up the caches in a random order. Ideally, this should not matter at all,
@@ -68,6 +69,7 @@ impl Caches {
             sourcefiles,
             bundle_index,
             diagnostics,
+            proguard,
         ];
         let mut rng = thread_rng();
         caches.as_mut_slice().shuffle(&mut rng);

crates/symbolicator-service/src/caching/config.rs

@@ -14,6 +14,7 @@ pub enum CacheName {
     SourceFiles,
     BundleIndex,
     Diagnostics,
+    Proguard,
 }
 
 impl AsRef<str> for CacheName {
@@ -30,6 +31,7 @@ impl AsRef<str> for CacheName {
             Self::SourceFiles => "sourcefiles",
             Self::BundleIndex => "bundle_index",
             Self::Diagnostics => "diagnostics",
+            Self::Proguard => "proguard",
         }
     }
 }