From 9a6005cb1dfead3a61c792a0c92aea54de47c542 Mon Sep 17 00:00:00 2001 From: Martin Bickel Date: Wed, 28 Aug 2024 17:10:19 +0200 Subject: [PATCH] Support exposing atttributes as labels Co-authored-by: Karina Calma --- .../java/io/prometheus/jmx/JmxCollector.java | 57 ++++++++++++++- .../java/io/prometheus/jmx/JmxScraper.java | 72 ++++++++++++++++++- docs/README.md | 13 ++++ 3 files changed, 136 insertions(+), 6 deletions(-) diff --git a/collector/src/main/java/io/prometheus/jmx/JmxCollector.java b/collector/src/main/java/io/prometheus/jmx/JmxCollector.java index a51568d5..dd63bd92 100644 --- a/collector/src/main/java/io/prometheus/jmx/JmxCollector.java +++ b/collector/src/main/java/io/prometheus/jmx/JmxCollector.java @@ -73,6 +73,16 @@ static class Rule { ArrayList labelValues; } + public static class MetricCustomizer { + MBeanFilter mbeanFilter; + List attributesAsLabels; + } + + public static class MBeanFilter { + String domain; + Map properties; + } + private static class Config { Integer startDelaySeconds = 0; String jmxUrl = ""; @@ -86,7 +96,7 @@ private static class Config { ObjectNameAttributeFilter objectNameAttributeFilter; List rules = new ArrayList<>(); long lastUpdate = 0L; - + List metricCustomizers = new ArrayList<>(); MatchedRulesCache rulesCache; } @@ -296,6 +306,29 @@ private Config loadConfig(Map yamlConfig) throws MalformedObject } } + if (yamlConfig.containsKey("metricCustomizers")) { + List> metricCustomizersYaml = + (List>) yamlConfig.get("metricCustomizers"); + for (Map metricCustomizerYaml : metricCustomizersYaml) { + Map mbeanFilterYaml = + (Map) metricCustomizerYaml.get("mbeanFilter"); + MBeanFilter mbeanFilter = new MBeanFilter(); + mbeanFilter.domain = (String) mbeanFilterYaml.get("domain"); + mbeanFilter.properties = (Map) mbeanFilterYaml.get("properties"); + + List attributesAsLabels = + (List) metricCustomizerYaml.get("attributesAsLabels"); + if (attributesAsLabels == null) { + attributesAsLabels = new ArrayList<>(); + } + + MetricCustomizer metricCustomizer = new MetricCustomizer(); + metricCustomizer.mbeanFilter = mbeanFilter; + metricCustomizer.attributesAsLabels = attributesAsLabels; + cfg.metricCustomizers.add(metricCustomizer); + } + } + if (yamlConfig.containsKey("rules")) { List> configRules = (List>) yamlConfig.get("rules"); @@ -469,7 +502,8 @@ private MatchedRule defaultExport( String help, Double value, double valueFactor, - String type) { + String type, + Map attributesAsLabelsWithValues) { StringBuilder name = new StringBuilder(); name.append(domain); if (beanProperties.size() > 0) { @@ -504,6 +538,7 @@ private MatchedRule defaultExport( labelValues.add(entry.getValue()); } } + addAttributesAsLabelsWithValuesToLabels(config, attributesAsLabelsWithValues, labelNames, labelValues); return new MatchedRule( fullname, matchName, type, help, labelNames, labelValues, value, valueFactor); @@ -512,6 +547,7 @@ private MatchedRule defaultExport( public void recordBean( String domain, LinkedHashMap beanProperties, + Map attributesAsLabelsWithValues, LinkedList attrKeys, String attrName, String attrType, @@ -609,7 +645,8 @@ public void recordBean( help, value, rule.valueFactor, - rule.type); + rule.type, + attributesAsLabelsWithValues); addToCache(rule, matchName, matchedRule); break; } @@ -631,6 +668,7 @@ public void recordBean( // Set the labels. ArrayList labelNames = new ArrayList<>(); ArrayList labelValues = new ArrayList<>(); + addAttributesAsLabelsWithValuesToLabels(config, attributesAsLabelsWithValues, labelNames, labelValues); if (rule.labelNames != null) { for (int i = 0; i < rule.labelNames.size(); i++) { final String unsafeLabelName = rule.labelNames.get(i); @@ -705,6 +743,18 @@ public void recordBean( } } + private static void addAttributesAsLabelsWithValuesToLabels(Config config, Map attributesAsLabelsWithValues, List labelNames, List labelValues) { + attributesAsLabelsWithValues.forEach( + (attributeAsLabelName, attributeValue) -> { + String labelName = safeName(attributeAsLabelName); + if (config.lowercaseOutputLabelNames) { + labelName = labelName.toLowerCase(); + } + labelNames.add(labelName); + labelValues.add(attributeValue); + }); + } + @Override public MetricSnapshots collect() { // Take a reference to the current config and collect with this one @@ -725,6 +775,7 @@ public MetricSnapshots collect() { config.includeObjectNames, config.excludeObjectNames, config.objectNameAttributeFilter, + config.metricCustomizers, receiver, jmxMBeanPropertyCache); diff --git a/collector/src/main/java/io/prometheus/jmx/JmxScraper.java b/collector/src/main/java/io/prometheus/jmx/JmxScraper.java index a1edecf1..67a4e46d 100644 --- a/collector/src/main/java/io/prometheus/jmx/JmxScraper.java +++ b/collector/src/main/java/io/prometheus/jmx/JmxScraper.java @@ -22,8 +22,24 @@ import io.prometheus.jmx.logger.LoggerFactory; import java.io.IOException; import java.lang.management.ManagementFactory; -import java.util.*; -import javax.management.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import javax.management.Attribute; +import javax.management.AttributeList; +import javax.management.JMException; +import javax.management.MBeanAttributeInfo; +import javax.management.MBeanInfo; +import javax.management.MBeanServerConnection; +import javax.management.ObjectInstance; +import javax.management.ObjectName; import javax.management.openmbean.CompositeData; import javax.management.openmbean.CompositeType; import javax.management.openmbean.TabularData; @@ -43,6 +59,7 @@ public interface MBeanReceiver { void recordBean( String domain, LinkedHashMap beanProperties, + Map attributesAsLabelsWithValues, LinkedList attrKeys, String attrName, String attrType, @@ -56,6 +73,7 @@ void recordBean( private final String password; private final boolean ssl; private final List includeObjectNames, excludeObjectNames; + private final List metricCustomizers; private final ObjectNameAttributeFilter objectNameAttributeFilter; private final JmxMBeanPropertyCache jmxMBeanPropertyCache; @@ -67,6 +85,7 @@ public JmxScraper( List includeObjectNames, List excludeObjectNames, ObjectNameAttributeFilter objectNameAttributeFilter, + List metricCustomizers, MBeanReceiver receiver, JmxMBeanPropertyCache jmxMBeanPropertyCache) { this.jmxUrl = jmxUrl; @@ -76,6 +95,7 @@ public JmxScraper( this.ssl = ssl; this.includeObjectNames = includeObjectNames; this.excludeObjectNames = excludeObjectNames; + this.metricCustomizers = metricCustomizers; this.objectNameAttributeFilter = objectNameAttributeFilter; this.jmxMBeanPropertyCache = jmxMBeanPropertyCache; } @@ -204,6 +224,12 @@ private void scrapeBean(MBeanServerConnection beanConn, ObjectName mBeanName) { final String mBeanNameString = mBeanName.toString(); final String mBeanDomain = mBeanName.getDomain(); + JmxCollector.MetricCustomizer metricCustomizer = getMetricCustomizer(mBeanName); + Map attributesAsLabelsWithValues = new HashMap<>(); + if (metricCustomizer != null) { + attributesAsLabelsWithValues = + getAttributesAsLabelsWithValues(metricCustomizer, attributes); + } for (Object object : attributes) { // The contents of an AttributeList should all be Attribute instances, but we'll verify @@ -231,6 +257,7 @@ private void scrapeBean(MBeanServerConnection beanConn, ObjectName mBeanName) { mBeanName, mBeanDomain, jmxMBeanPropertyCache.getKeyPropertyList(mBeanName), + attributesAsLabelsWithValues, new LinkedList<>(), mBeanAttributeInfo.getName(), mBeanAttributeInfo.getType(), @@ -251,6 +278,35 @@ private void scrapeBean(MBeanServerConnection beanConn, ObjectName mBeanName) { } } + private Map getAttributesAsLabelsWithValues(JmxCollector.MetricCustomizer metricCustomizer, AttributeList attributes) { + Map attributeMap = attributes.asList().stream() + .collect(Collectors.toMap(Attribute::getName, Attribute::getValue)); + Map attributesAsLabelsWithValues = new HashMap<>(); + for (String attributeAsLabel : metricCustomizer.attributesAsLabels) { + Object attrValue = attributeMap.get(attributeAsLabel); + if (attrValue != null) { + attributesAsLabelsWithValues.put(attributeAsLabel, attrValue.toString()); + } + } + return attributesAsLabelsWithValues; + } + + private JmxCollector.MetricCustomizer getMetricCustomizer(ObjectName mBeanName) { + if (!metricCustomizers.isEmpty()) { + for (JmxCollector.MetricCustomizer metricCustomizer : metricCustomizers) { + if (filterMbeanByDomainAndProperties(mBeanName, metricCustomizer)) { + return metricCustomizer; + } + } + } + return null; + } + + private boolean filterMbeanByDomainAndProperties(ObjectName mBeanName, JmxCollector.MetricCustomizer metricCustomizer) { + return metricCustomizer.mbeanFilter.domain.equals(mBeanName.getDomain()) && + mBeanName.getKeyPropertyList().entrySet().containsAll(metricCustomizer.mbeanFilter.properties.entrySet()); + } + private void processAttributesOneByOne( MBeanServerConnection beanConn, ObjectName mbeanName, @@ -269,6 +325,7 @@ private void processAttributesOneByOne( mbeanName, mbeanName.getDomain(), jmxMBeanPropertyCache.getKeyPropertyList(mbeanName), + new HashMap<>(), new LinkedList<>(), attr.getName(), attr.getType(), @@ -286,6 +343,7 @@ private void processBeanValue( ObjectName objectName, String domain, LinkedHashMap beanProperties, + Map attributesAsLabelsWithValues, LinkedList attrKeys, String attrName, String attrType, @@ -303,7 +361,7 @@ private void processBeanValue( } LOGGER.log(FINE, "%s%s%s scrape: %s", domain, beanProperties, attrName, value); this.receiver.recordBean( - domain, beanProperties, attrKeys, attrName, attrType, attrDescription, value); + domain, beanProperties, attributesAsLabelsWithValues, attrKeys, attrName, attrType, attrDescription, value); } else if (value instanceof CompositeData) { LOGGER.log(FINE, "%s%s%s scrape: compositedata", domain, beanProperties, attrName); CompositeData composite = (CompositeData) value; @@ -317,6 +375,7 @@ private void processBeanValue( objectName, domain, beanProperties, + attributesAsLabelsWithValues, attrKeys, key, typ, @@ -383,6 +442,7 @@ private void processBeanValue( objectName, domain, l2s, + attributesAsLabelsWithValues, attrNames, name, typ, @@ -403,6 +463,7 @@ private void processBeanValue( objectName, domain, beanProperties, + attributesAsLabelsWithValues, attrKeys, attrName, attrType, @@ -415,6 +476,7 @@ private void processBeanValue( objectName, domain, beanProperties, + attributesAsLabelsWithValues, attrKeys, attrName, attrType, @@ -430,6 +492,7 @@ private static class StdoutWriter implements MBeanReceiver { public void recordBean( String domain, LinkedHashMap beanProperties, + Map attributesAsLabelsWithValues, LinkedList attrKeys, String attrName, String attrType, @@ -454,6 +517,7 @@ public static void main(String[] args) throws Exception { objectNames, new LinkedList<>(), objectNameAttributeFilter, + new LinkedList<>(), new StdoutWriter(), new JmxMBeanPropertyCache()) .doScrape(); @@ -466,6 +530,7 @@ public static void main(String[] args) throws Exception { objectNames, new LinkedList<>(), objectNameAttributeFilter, + new LinkedList<>(), new StdoutWriter(), new JmxMBeanPropertyCache()) .doScrape(); @@ -478,6 +543,7 @@ public static void main(String[] args) throws Exception { objectNames, new LinkedList<>(), objectNameAttributeFilter, + new LinkedList<>(), new StdoutWriter(), new JmxMBeanPropertyCache()) .doScrape(); diff --git a/docs/README.md b/docs/README.md index 843dc6b5..27c087e3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -109,6 +109,14 @@ excludeObjectNameAttributes: "java.lang:type=Runtime": - "ClassPath" - "SystemProperties" +metricCustomizers: + - mbeanFilter: + domain: org.apache.cassandra.metrics + properties: + type: + attributesAsLabels: + - string1 + - string2 rules: - pattern: 'org.apache.cassandra.metrics<>Value: (\d+)' name: cassandra_$1_$2 @@ -119,6 +127,7 @@ rules: cache: false type: GAUGE attrNameSnakeCase: false + ``` Name | Description ---------|------------ @@ -144,6 +153,10 @@ labels | A map of label name to label value pairs. Capture groups fro help | Help text for the metric. Capture groups from `pattern` can be used. `name` must be set to use this. Defaults to the mBean attribute description, domain, and name of the attribute. cache | Whether to cache bean name expressions to rule computation (match and mismatch). Not recommended for rules matching on bean value, as only the value from the first scrape will be cached and re-used. This can increase performance when collecting a lot of mbeans. Defaults to `false`. type | The type of the metric, can be `GAUGE`, `COUNTER` or `UNTYPED`. `name` must be set to use this. Defaults to `UNTYPED`. +metricCustomizers | A list of objects that contain `mbeanFilter` and `attributesAsLabels`. For those mbeans that match the filter, the items in the `attributesAsLabels` list will be added as attributes to the existing metrics. +domain | Domain of an mbean. +properties | Properties of an mbean. +attributesAsLabels | List of elements to be added as attributes to existing metrics. Metric names and label names are sanitized. All characters other than `[a-zA-Z0-9:_]` are replaced with underscores, and adjacent underscores are collapsed. There's no limitations on label values or the help text.