feat(ourlogs): Allow log ingestion behind a flag

CHANGELOG.md

@@ -1,10 +1,14 @@
 # Changelog
 
+## Unreleased
+- Add data categories for LogItem and LogByte. ([#4448](https://github.com/getsentry/relay/pull/4455))
+
 ## 25.1.0
 
 **Internal**
 
 - Updates performance score calculation on spans and events to also store cdf values as measurements. ([#4438](https://github.com/getsentry/relay/pull/4438))
+- Allow log ingestion behind a flag, only for internal use currently. ([#4448](https://github.com/getsentry/relay/pull/4448))
 
 ## 24.12.2
 

Cargo.toml

@@ -43,6 +43,7 @@ relay-kafka = { path = "relay-kafka" }
 relay-log = { path = "relay-log" }
 relay-metrics = { path = "relay-metrics" }
 relay-monitors = { path = "relay-monitors" }
+relay-ourlogs = { path = "relay-ourlogs" }
 relay-pattern = { path = "relay-pattern" }
 relay-pii = { path = "relay-pii" }
 relay-profiling = { path = "relay-profiling" }
@@ -150,7 +151,7 @@ reqwest = "0.12.9"
 rmp-serde = "1.3.0"
 sentry = "0.34.0"
 sentry-core = "0.34.0"
-sentry-kafka-schemas = { version = "0.1.122", default-features = false }
+sentry-kafka-schemas = { version = "0.1.129", default-features = false }
 sentry-release-parser = { version = "1.3.2", default-features = false }
 sentry-types = "0.34.0"
 semver = "1.0.23"

py/sentry_relay/consts.py

@@ -8,7 +8,7 @@
 
 
 class DataCategory(IntEnum):
-    # begin generated
+    # start generated
     DEFAULT = 0
     ERROR = 1
     TRANSACTION = 2
@@ -32,6 +32,8 @@ class DataCategory(IntEnum):
     REPLAY_VIDEO = 20
     UPTIME = 21
     ATTACHMENT_ITEM = 22
+    LOG_ITEM = 23
+    LOG_BYTE = 24
     UNKNOWN = -1
     # end generated
 

relay-base-schema/src/data_category.rs

@@ -92,6 +92,15 @@ pub enum DataCategory {
     Uptime = 21,
     /// Counts the number of individual attachments, as opposed to the number of bytes in an attachment.
     AttachmentItem = 22,
+    /// LogItem
+    ///
+    /// This is the category for logs for which we store the count log events for users for measuring
+    /// missing breadcrumbs, and count of logs for rate limiting purposes.
+    LogItem = 23,
+    /// LogByte
+    ///
+    /// This is the category for logs for which we store log event total bytes for users.
+    LogByte = 24,
     //
     // IMPORTANT: After adding a new entry to DataCategory, go to the `relay-cabi` subfolder and run
     // `make header` to regenerate the C-binding. This allows using the data category from Python.
@@ -120,6 +129,8 @@ impl DataCategory {
             "transaction_indexed" => Self::TransactionIndexed,
             "monitor" => Self::Monitor,
             "span" => Self::Span,
+            "log_item" => Self::LogItem,
+            "log_byte" => Self::LogByte,
             "monitor_seat" => Self::MonitorSeat,
             "feedback" => Self::UserReportV2,
             "user_report_v2" => Self::UserReportV2,
@@ -152,6 +163,8 @@ impl DataCategory {
             Self::TransactionIndexed => "transaction_indexed",
             Self::Monitor => "monitor",
             Self::Span => "span",
+            Self::LogItem => "log_item",
+            Self::LogByte => "log_byte",
             Self::MonitorSeat => "monitor_seat",
             Self::UserReportV2 => "feedback",
             Self::MetricBucket => "metric_bucket",

relay-cabi/include/relay.h

@@ -3,7 +3,7 @@
 #ifndef RELAY_H_INCLUDED
 #define RELAY_H_INCLUDED
 
-/* Generated with cbindgen:0.26.0 */
+/* Generated with cbindgen:0.27.0 */
 
 /* Warning, this file is autogenerated. Do not modify this manually. */
 
@@ -142,6 +142,19 @@ enum RelayDataCategory {
    * Counts the number of individual attachments, as opposed to the number of bytes in an attachment.
    */
   RELAY_DATA_CATEGORY_ATTACHMENT_ITEM = 22,
+  /**
+   * LogItem
+   *
+   * This is the category for logs for which we store the count log events for users for measuring
+   * missing breadcrumbs, and count of logs for rate limiting purposes.
+   */
+  RELAY_DATA_CATEGORY_LOG_ITEM = 23,
+  /**
+   * LogByte
+   *
+   * This is the category for logs for which we store log event total bytes for users.
+   */
+  RELAY_DATA_CATEGORY_LOG_BYTE = 24,
   /**
    * Any other data category not known by this Relay.
    */
@@ -679,4 +692,4 @@ struct RelayStr normalize_cardinality_limit_config(const struct RelayStr *value)
  */
 struct RelayStr relay_normalize_global_config(const struct RelayStr *value);
 
-#endif /* RELAY_H_INCLUDED */
+#endif  /* RELAY_H_INCLUDED */

relay-cogs/src/lib.rs

@@ -117,6 +117,8 @@ pub enum AppFeature {
     Transactions,
     /// Errors.
     Errors,
+    /// Logs.
+    Logs,
     /// Spans.
     Spans,
     /// Sessions.
@@ -159,6 +161,7 @@ impl AppFeature {
             Self::Transactions => "transactions",
             Self::Errors => "errors",
             Self::Spans => "spans",
+            Self::Logs => "our_logs",
             Self::Sessions => "sessions",
             Self::ClientReports => "client_reports",
             Self::CheckIns => "check_ins",

relay-config/src/config.rs

@@ -617,6 +617,8 @@ pub struct Limits {
     /// The maximum payload size for a profile
     pub max_profile_size: ByteSize,
     /// The maximum payload size for a span.
+    pub max_log_size: ByteSize,
+    /// The maximum payload size for a span.
     pub max_span_size: ByteSize,
     /// The maximum payload size for a statsd metric.
     pub max_statsd_size: ByteSize,
@@ -677,6 +679,7 @@ impl Default for Limits {
             max_api_file_upload_size: ByteSize::mebibytes(40),
             max_api_chunk_upload_size: ByteSize::mebibytes(100),
             max_profile_size: ByteSize::mebibytes(50),
+            max_log_size: ByteSize::mebibytes(1),
             max_span_size: ByteSize::mebibytes(1),
             max_statsd_size: ByteSize::mebibytes(1),
             max_metric_buckets_size: ByteSize::mebibytes(1),
@@ -2206,6 +2209,11 @@ impl Config {
         self.values.limits.max_check_in_size.as_bytes()
     }
 
+    /// Returns the maximum payload size of a log in bytes.
+    pub fn max_log_size(&self) -> usize {
+        self.values.limits.max_log_size.as_bytes()
+    }
+
     /// Returns the maximum payload size of a span in bytes.
     pub fn max_span_size(&self) -> usize {
         self.values.limits.max_span_size.as_bytes()

relay-dynamic-config/src/feature.rs

@@ -102,7 +102,11 @@ pub enum Feature {
     /// Serialized as `organizations:ingest-spans-in-eap`
     #[serde(rename = "organizations:ingest-spans-in-eap")]
     IngestSpansInEap,
-
+    /// Enable log ingestion for our log product (this is not internal logging).
+    ///
+    /// Serialized as `organizations:ourlogs-ingestion`.
+    #[serde(rename = "organizations:ourlogs-ingestion")]
+    OurLogsIngestion,
     /// This feature has graduated and is hard-coded for external Relays.
     #[doc(hidden)]
     #[serde(rename = "projects:profiling-ingest-unsampled-profiles")]

relay-event-schema/src/processor/attrs.rs

@@ -46,6 +46,7 @@ pub enum ValueType {
     Message,
     Thread,
     Breadcrumb,
+    OurLog,
     Span,
     ClientSdkInfo,
 
@@ -84,6 +85,7 @@ relay_common::derive_fromstr_and_display!(ValueType, UnknownValueTypeError, {
     ValueType::Message => "message",
     ValueType::Thread => "thread",
     ValueType::Breadcrumb => "breadcrumb",
+    ValueType::OurLog => "ourlog",
     ValueType::Span => "span",
     ValueType::ClientSdkInfo => "sdk",
     ValueType::Minidump => "minidump",

relay-event-schema/src/processor/traits.rs

@@ -108,10 +108,12 @@ pub trait Processor: Sized {
     process_method!(process_breadcrumb, crate::protocol::Breadcrumb);
     process_method!(process_template_info, crate::protocol::TemplateInfo);
     process_method!(process_header_name, crate::protocol::HeaderName);
+    process_method!(process_ourlog, crate::protocol::OurLog);
     process_method!(process_span, crate::protocol::Span);
     process_method!(process_trace_context, crate::protocol::TraceContext);
     process_method!(process_native_image_path, crate::protocol::NativeImagePath);
     process_method!(process_contexts, crate::protocol::Contexts);
+    process_method!(process_attribute_value, crate::protocol::AttributeValue);
 
     fn process_other(
         &mut self,

relay-event-schema/src/protocol/mod.rs

@@ -18,6 +18,7 @@ mod mechanism;
 mod metrics;
 mod metrics_summary;
 mod nel;
+mod ourlog;
 mod relay_info;
 mod replay;
 mod request;
@@ -54,6 +55,7 @@ pub use self::mechanism::*;
 pub use self::metrics::*;
 pub use self::metrics_summary::*;
 pub use self::nel::*;
+pub use self::ourlog::*;
 pub use self::relay_info::*;
 pub use self::replay::*;
 pub use self::request::*;