feat(ourlogs): Allow log ingestion behind a flag

CHANGELOG.md

@@ -5,6 +5,7 @@
 **Internal**
 
 - Updates performance score calculation on spans and events to also store cdf values as measurements. ([#4438](https://github.com/getsentry/relay/pull/4438))
+- Allow log ingestion behind a flag, only for internal use currently. ([#4448](https://github.com/getsentry/relay/pull/4448))
 
 ## 24.12.2
 

Cargo.toml

@@ -43,6 +43,7 @@ relay-kafka = { path = "relay-kafka" }
 relay-log = { path = "relay-log" }
 relay-metrics = { path = "relay-metrics" }
 relay-monitors = { path = "relay-monitors" }
+relay-ourlogs = { path = "relay-ourlogs" }
 relay-pattern = { path = "relay-pattern" }
 relay-pii = { path = "relay-pii" }
 relay-profiling = { path = "relay-profiling" }

py/sentry_relay/consts.py

@@ -8,7 +8,6 @@
 
 
 class DataCategory(IntEnum):
-    # begin generated
     DEFAULT = 0
     ERROR = 1
     TRANSACTION = 2
@@ -32,6 +31,8 @@ class DataCategory(IntEnum):
     REPLAY_VIDEO = 20
     UPTIME = 21
     ATTACHMENT_ITEM = 22
+    LOG_COUNT = 23
+    LOG_BYTES = 24
     UNKNOWN = -1
     # end generated
 

relay-base-schema/src/data_category.rs

@@ -92,6 +92,15 @@ pub enum DataCategory {
     Uptime = 21,
     /// Counts the number of individual attachments, as opposed to the number of bytes in an attachment.
     AttachmentItem = 22,
+    /// LogCount
+    ///
+    /// This is the category for logs for which we store the count log events for users for measuring
+    /// missing breadcrumbs, and count of logs for rate limiting purposes.
+    LogCount = 23,
+    /// LogBytes
+    ///
+    /// This is the category for logs for which we store log event total bytes for users.
+    LogBytes = 24,
     //
     // IMPORTANT: After adding a new entry to DataCategory, go to the `relay-cabi` subfolder and run
     // `make header` to regenerate the C-binding. This allows using the data category from Python.
@@ -120,6 +129,8 @@ impl DataCategory {
             "transaction_indexed" => Self::TransactionIndexed,
             "monitor" => Self::Monitor,
             "span" => Self::Span,
+            "log_count" => Self::LogCount,
+            "log_bytes" => Self::LogBytes,
             "monitor_seat" => Self::MonitorSeat,
             "feedback" => Self::UserReportV2,
             "user_report_v2" => Self::UserReportV2,
@@ -152,6 +163,8 @@ impl DataCategory {
             Self::TransactionIndexed => "transaction_indexed",
             Self::Monitor => "monitor",
             Self::Span => "span",
+            Self::LogCount => "log_count",
+            Self::LogBytes => "log_bytes",
             Self::MonitorSeat => "monitor_seat",
             Self::UserReportV2 => "feedback",
             Self::MetricBucket => "metric_bucket",

relay-cabi/include/relay.h

@@ -3,7 +3,7 @@
 #ifndef RELAY_H_INCLUDED
 #define RELAY_H_INCLUDED
 
-/* Generated with cbindgen:0.26.0 */
+/* Generated with cbindgen:0.27.0 */
 
 /* Warning, this file is autogenerated. Do not modify this manually. */
 
@@ -142,6 +142,19 @@ enum RelayDataCategory {
    * Counts the number of individual attachments, as opposed to the number of bytes in an attachment.
    */
   RELAY_DATA_CATEGORY_ATTACHMENT_ITEM = 22,
+  /**
+   * LogCount
+   *
+   * This is the category for logs for which we store the count log events for users for measuring
+   * missing breadcrumbs, and count of logs for rate limiting purposes.
+   */
+  RELAY_DATA_CATEGORY_LOG_COUNT = 23,
+  /**
+   * LogBytes
+   *
+   * This is the category for logs for which we store log event total bytes for users.
+   */
+  RELAY_DATA_CATEGORY_LOG_BYTES = 24,
   /**
    * Any other data category not known by this Relay.
    */
@@ -679,4 +692,4 @@ struct RelayStr normalize_cardinality_limit_config(const struct RelayStr *value)
  */
 struct RelayStr relay_normalize_global_config(const struct RelayStr *value);
 
-#endif /* RELAY_H_INCLUDED */
+#endif  /* RELAY_H_INCLUDED */

relay-cogs/src/lib.rs

@@ -117,6 +117,8 @@ pub enum AppFeature {
     Transactions,
     /// Errors.
     Errors,
+    /// Logs.
+    Logs,
     /// Spans.
     Spans,
     /// Sessions.
@@ -159,6 +161,7 @@ impl AppFeature {
             Self::Transactions => "transactions",
             Self::Errors => "errors",
             Self::Spans => "spans",
+            Self::Logs => "our_logs",
             Self::Sessions => "sessions",
             Self::ClientReports => "client_reports",
             Self::CheckIns => "check_ins",

relay-config/src/config.rs

@@ -617,6 +617,8 @@ pub struct Limits {
     /// The maximum payload size for a profile
     pub max_profile_size: ByteSize,
     /// The maximum payload size for a span.
+    pub max_log_size: ByteSize,
+    /// The maximum payload size for a span.
     pub max_span_size: ByteSize,
     /// The maximum payload size for a statsd metric.
     pub max_statsd_size: ByteSize,
@@ -677,6 +679,7 @@ impl Default for Limits {
             max_api_file_upload_size: ByteSize::mebibytes(40),
             max_api_chunk_upload_size: ByteSize::mebibytes(100),
             max_profile_size: ByteSize::mebibytes(50),
+            max_log_size: ByteSize::mebibytes(1),
             max_span_size: ByteSize::mebibytes(1),
             max_statsd_size: ByteSize::mebibytes(1),
             max_metric_buckets_size: ByteSize::mebibytes(1),
@@ -2206,6 +2209,11 @@ impl Config {
         self.values.limits.max_check_in_size.as_bytes()
     }
 
+    /// Returns the maximum payload size of a log in bytes.
+    pub fn max_log_size(&self) -> usize {
+        self.values.limits.max_log_size.as_bytes()
+    }
+
     /// Returns the maximum payload size of a span in bytes.
     pub fn max_span_size(&self) -> usize {
         self.values.limits.max_span_size.as_bytes()

relay-dynamic-config/src/feature.rs

@@ -102,7 +102,11 @@ pub enum Feature {
     /// Serialized as `organizations:ingest-spans-in-eap`
     #[serde(rename = "organizations:ingest-spans-in-eap")]
     IngestSpansInEap,
-
+    /// Enable log ingestion for our log product (this is not internal logging).
+    ///
+    /// Serialized as `organizations:ourlogs-ingestion`.
+    #[serde(rename = "organizations:ourlogs-ingestion")]
+    OurLogsIngestion,
     /// This feature has graduated and is hard-coded for external Relays.
     #[doc(hidden)]
     #[serde(rename = "projects:profiling-ingest-unsampled-profiles")]

relay-event-schema/src/processor/attrs.rs

@@ -46,6 +46,7 @@ pub enum ValueType {
     Message,
     Thread,
     Breadcrumb,
+    OurLog,
     Span,
     ClientSdkInfo,
 
@@ -84,6 +85,7 @@ relay_common::derive_fromstr_and_display!(ValueType, UnknownValueTypeError, {
     ValueType::Message => "message",
     ValueType::Thread => "thread",
     ValueType::Breadcrumb => "breadcrumb",
+    ValueType::OurLog => "ourlog",
     ValueType::Span => "span",
     ValueType::ClientSdkInfo => "sdk",
     ValueType::Minidump => "minidump",

relay-event-schema/src/processor/traits.rs

@@ -108,6 +108,7 @@ pub trait Processor: Sized {
     process_method!(process_breadcrumb, crate::protocol::Breadcrumb);
     process_method!(process_template_info, crate::protocol::TemplateInfo);
     process_method!(process_header_name, crate::protocol::HeaderName);
+    process_method!(process_ourlog, crate::protocol::OurLog);
     process_method!(process_span, crate::protocol::Span);
     process_method!(process_trace_context, crate::protocol::TraceContext);
     process_method!(process_native_image_path, crate::protocol::NativeImagePath);

relay-event-schema/src/protocol/mod.rs

@@ -18,6 +18,7 @@ mod mechanism;
 mod metrics;
 mod metrics_summary;
 mod nel;
+mod ourlog;
 mod relay_info;
 mod replay;
 mod request;
@@ -54,6 +55,7 @@ pub use self::mechanism::*;
 pub use self::metrics::*;
 pub use self::metrics_summary::*;
 pub use self::nel::*;
+pub use self::ourlog::*;
 pub use self::relay_info::*;
 pub use self::replay::*;
 pub use self::request::*;

relay-event-schema/src/protocol/ourlog.rs

@@ -0,0 +1,138 @@
+use relay_protocol::{Annotated, Empty, FromValue, IntoValue, Object, Value};
+
+use crate::processor::ProcessValue;
+use crate::protocol::{SpanId, TraceId};
+
+#[derive(Clone, Debug, Default, PartialEq, Empty, FromValue, IntoValue, ProcessValue)]
+#[metastructure(process_func = "process_ourlog", value_type = "OurLog")]
+pub struct OurLog {
+    /// Time when the event occurred.
+    #[metastructure(required = true, trim = false)]
+    pub timestamp_nanos: Annotated<u64>,
+
+    /// Time when the event was observed.
+    #[metastructure(required = true, trim = false)]
+    pub observed_timestamp_nanos: Annotated<u64>,
+
+    /// The ID of the trace the log belongs to.
+    #[metastructure(required = false, trim = false)]
+    pub trace_id: Annotated<TraceId>,
+    /// The Span id.
+    ///
+    #[metastructure(required = false, trim = false)]
+    pub span_id: Annotated<SpanId>,
+
+    /// Trace flag bitfield.
+    #[metastructure(required = false)]
+    pub trace_flags: Annotated<f64>,
+
+    /// This is the original string representation of the severity as it is known at the source
+    #[metastructure(required = false, max_chars = 32, pii = "maybe", trim = false)]
+    pub severity_text: Annotated<String>,
+
+    /// Numerical representation of the severity level
+    #[metastructure(required = false)]
+    pub severity_number: Annotated<i64>,
+
+    /// Log body.
+    #[metastructure(required = true, pii = "maybe", trim = false)]
+    pub body: Annotated<String>,
+
+    /// Arbitrary attributes on a log.
+    #[metastructure(pii = "maybe", trim = false)]
+    pub attributes: Annotated<Object<AttributeValue>>,
+
+    /// Additional arbitrary fields for forwards compatibility.
+    #[metastructure(additional_properties, retain = true, pii = "maybe", trim = false)]
+    pub other: Object<Value>,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Empty, FromValue, IntoValue, ProcessValue)]
+pub struct AttributeValue {
+    pub string_value: Annotated<Value>,
+    pub int_value: Annotated<Value>,
+    pub double_value: Annotated<Value>,
+    pub bool_value: Annotated<Value>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_ourlog_serialization() {
+        let json = r#"{
+  "timestamp_nanos": 1544712660300000000,
+  "observed_timestamp_nanos": 1544712660300000000,
+  "severity_number": 10,
+  "severity_text": "Information",
+  "trace_id": "5b8efff798038103d269b633813fc60c",
+  "span_id": "eee19b7ec3c1b174",
+  "body": "Example log record",
+  "attributes": {
+    "string.attribute": {
+      "string_value": "some string"
+    },
+    "boolean.attribute": {
+      "bool_value": true
+    },
+    "int.attribute": {
+      "int_value": 10
+    },
+    "double.attribute": {
+      "double_value": 637.704
+    }
+  }
+}"#;
+
+        let mut attributes = Object::new();
+        attributes.insert(
+            "string.attribute".into(),
+            Annotated::new(AttributeValue {
+                string_value: Annotated::new(Value::String("some string".into())),
+                ..Default::default()
+            }),
+        );
+        attributes.insert(
+            "boolean.attribute".into(),
+            Annotated::new(AttributeValue {
+                bool_value: Annotated::new(Value::Bool(true)),
+                ..Default::default()
+            }),
+        );
+        attributes.insert(
+            "int.attribute".into(),
+            Annotated::new(AttributeValue {
+                int_value: Annotated::new(Value::I64(10)),
+                ..Default::default()
+            }),
+        );
+        attributes.insert(
+            "double.attribute".into(),
+            Annotated::new(AttributeValue {
+                double_value: Annotated::new(Value::F64(637.704)),
+                ..Default::default()
+            }),
+        );
+
+        let log = Annotated::new(OurLog {
+            timestamp_nanos: Annotated::new(1544712660300000000),
+            observed_timestamp_nanos: Annotated::new(1544712660300000000),
+            severity_number: Annotated::new(10),
+            severity_text: Annotated::new("Information".to_string()),
+            trace_id: Annotated::new(TraceId("5b8efff798038103d269b633813fc60c".into())),
+            span_id: Annotated::new(SpanId("eee19b7ec3c1b174".into())),
+            body: Annotated::new("Example log record".to_string()),
+            attributes: Annotated::new(attributes),
+            ..Default::default()
+        });
+
+        let expected: serde_json::Value = serde_json::from_str(json).unwrap();
+        let actual: serde_json::Value =
+            serde_json::from_str(&log.to_json_pretty().unwrap()).unwrap();
+        assert_eq!(expected, actual);
+
+        let log_from_string = Annotated::<OurLog>::from_json(json).unwrap();
+        assert_eq!(log, log_from_string);
+    }
+}