diff --git a/public/models/interfaces.ts b/public/models/interfaces.ts index 18044e38..e2eea176 100644 --- a/public/models/interfaces.ts +++ b/public/models/interfaces.ts @@ -234,14 +234,23 @@ export type AnomalyData = { plotTime?: number; entity?: EntityData[]; features?: { [key: string]: FeatureAggregationData }; - aggInterval?: string; + contributions?: { [key: string]: FeatureContributionData }; + aggInterval?: string; }; export type FeatureAggregationData = { data: number; + name?: string; endTime: number; startTime: number; plotTime?: number; + attribution?: number; + expectedValue?: number; +}; + +export type FeatureContributionData = { + name: string; + attribution: number; }; export type Anomalies = { @@ -279,6 +288,7 @@ export type AnomalySummary = { minConfidence: number; maxConfidence: number; lastAnomalyOccurrence: string; + contributions?: string; }; export type DateRange = { diff --git a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx index 2647d386..ae9b47ec 100644 --- a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx +++ b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx @@ -205,6 +205,7 @@ export const FeatureChart = (props: FeatureChartProps) => { * and thus show different annotations per feature chart (currently all annotations * shown equally across all enabled feature charts for a given detector). */} + {props.feature.featureEnabled ? ( { } {featureData.map( (featureTimeSeries: FeatureAggregationData[], index) => { + const timeSeriesList: any[] = []; const seriesKey = props.isHCDetector ? `${props.featureDataSeriesName} (${convertToEntityString( props.entityData[index], ', ' )})` : props.featureDataSeriesName; - return ( + timeSeriesList.push( { yAccessors={[CHART_FIELDS.DATA]} data={featureTimeSeries} /> - ); + ) + if (featureTimeSeries.map( + (item: FeatureAggregationData) => { + if(item.hasOwnProperty('expectedValue')) { + timeSeriesList.push( + + ) + } + } + )) + return timeSeriesList; } )} + {showCustomExpression ? ( state.anomalyResults.requesting ); @@ -344,6 +348,69 @@ export const AnomalyDetailsChart = React.memo( zoomRange.endDate, ]); + + const customAnomalyContributionTooltip = (details?: string) => { + const anomaly = details ? JSON.parse(details) : undefined; + const contributionData = get(anomaly, `contributions`, []) + + const featureData = get(anomaly, `features`, {}) + let featureAttributionList = [] as any[]; + if (Array.isArray(contributionData)) { + contributionData.map((contribution: any) => { + const featureName = get(get(featureData, contribution.feature_id, ""), "name", "") + const dataString = (contribution.data * 100) + "%" + featureAttributionList.push( +
+ {featureName}: {dataString}
+
+ ) + }) + } else { + for (const [, value] of Object.entries(contributionData)) { + featureAttributionList.push( +
+ {value.name}: {value.attribution}
+
+ ) + } + } + return ( +
+ + Feature Contribution: + {anomaly ? ( +

+

+ {featureAttributionList} +

+ ) : null} + +
+ ); + }; + + + const generateContributionAnomalyAnnotations = ( + anomalies: AnomalyData[][] + ): any[][] => { + let annotations = [] as any[]; + anomalies.forEach((anomalyTimeSeries: AnomalyData[]) => { + annotations.push( + anomalyTimeSeries + .filter((anomaly: AnomalyData) => anomaly.anomalyGrade > 0) + .map((anomaly: AnomalyData) => ( + { + coordinates: { + x0: anomaly.startTime, + x1: anomaly.endTime + (anomaly.endTime - anomaly.startTime), + }, + details: `${JSON.stringify(anomaly)}` + })) + ); + }); + return annotations; + }; + const isLoading = props.isLoading || isLoadingAlerts || isRequestingAnomalyResults; const isInitializingHistorical = taskState === DETECTOR_STATE.INIT; @@ -531,7 +598,18 @@ export const AnomalyDetailsChart = React.memo( }} /> )} - + + {alertAnnotations ? ( - ); + : props.anomalyGradeSeriesName; + return ( + + ) } )} diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx index b2603ada..23c7031d 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx @@ -640,7 +640,9 @@ export const getFeatureBreakdownWording = ( return isNotSample ? 'Feature breakdown' : 'Sample feature breakdown'; }; -export const getFeatureDataWording = (isNotSample: boolean | undefined) => { +export const getFeatureDataWording = ( + isNotSample: boolean | undefined +) => { return isNotSample ? 'Feature output' : 'Sample feature output'; }; diff --git a/public/pages/AnomalyCharts/utils/constants.ts b/public/pages/AnomalyCharts/utils/constants.ts index 768d32f8..5b699a1b 100644 --- a/public/pages/AnomalyCharts/utils/constants.ts +++ b/public/pages/AnomalyCharts/utils/constants.ts @@ -24,11 +24,13 @@ export enum CHART_FIELDS { CONFIDENCE = 'confidence', DATA = 'data', AGG_INTERVAL = 'aggInterval', + EXPECTED_VALUE = 'expectedValue' } export enum CHART_COLORS { ANOMALY_GRADE_COLOR = '#D13212', FEATURE_DATA_COLOR = '#16191F', + FEATURE_COLOR = '#fcd529', CONFIDENCE_COLOR = '#017F75', LIGHT_BACKGROUND = '#FFFFFF', DARK_BACKGROUND = '#1D1E24', @@ -89,7 +91,7 @@ export const DEFAULT_ANOMALY_SUMMARY = { maxAnomalyGrade: 0, minConfidence: 0, maxConfidence: 0, - lastAnomalyOccurrence: '-', + lastAnomalyOccurrence: '-' }; export const HEATMAP_CHART_Y_AXIS_WIDTH = 30; diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index 2c8c5c0f..fcdbdda5 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -136,7 +136,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { endDate: initialEndDate, }); const [selectedTabId, setSelectedTabId] = useState( - ANOMALY_HISTORY_TABS.ANOMALY_OCCURRENCE + ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN ); const [isLoadingAnomalyResults, setIsLoadingAnomalyResults] = useState(false); @@ -425,7 +425,6 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { setIsLoadingAnomalyResults(false); errorFetchingResults = true; }); - const rawAnomaliesData = get(detectorResultResponse, 'response', []); const rawAnomaliesResult = { anomalies: get(rawAnomaliesData, 'results', []), @@ -610,6 +609,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { fetchBucketizedEntityAnomalyData(entityLists); } else { fetchAllEntityAnomalyData(dateRange, entityLists); + // no visit setBucketizedAnomalyResults(undefined); } } catch (err) { @@ -807,13 +807,13 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { const tabs = [ { - id: ANOMALY_HISTORY_TABS.ANOMALY_OCCURRENCE, - name: 'Anomaly occurrences', + id: ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN, + name: 'Feature breakdown', disabled: false, }, { - id: ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN, - name: 'Feature breakdown', + id: ANOMALY_HISTORY_TABS.ANOMALY_OCCURRENCE, + name: 'Anomaly occurrences', disabled: false, }, ]; diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index b5f5a327..55f907a6 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -44,6 +44,7 @@ import { Detector, FeatureAggregationData, FeatureAttributes, + FeatureContributionData, MonitorAlert, toDuration } from '../../models/interfaces'; @@ -375,6 +376,8 @@ export const RETURNED_AD_RESULT_FIELDS = [ 'confidence', 'feature_data', 'entity', + 'relevant_attribution', + 'expected_values' ]; export const getAnomalySummaryQuery = ( @@ -575,6 +578,23 @@ export const parseBucketizedAnomalyResults = (result: any): Anomalies => { get(rawAnomaly, 'anomaly_grade') !== undefined && get(rawAnomaly, 'feature_data', []).length > 0 ) { + const featureDataMap: {[key: string]: string} = {}; + get(rawAnomaly, 'feature_data', []).filter(element => get(element, 'feature_id') && get(element, 'feature_name')). + forEach((feature: any) => { + featureDataMap[get(feature, 'feature_id')] = get(feature, 'feature_name'); + }); + const relevantAttributionMap: {[key: string]: number} = {}; + get(rawAnomaly, 'relevant_attribution', []).filter(element => get(element, 'feature_id') && get(element, 'data')). + forEach((attribution: any) => { + relevantAttributionMap[get(attribution, 'feature_id')] = get(attribution, 'data'); + }); + const contributions: { [key: string]: FeatureContributionData } = {} + for (const [key] of Object.entries(featureDataMap)) { + contributions[key] = { + name: featureDataMap[key], + attribution: relevantAttributionMap[key] + } + } anomalies.push({ anomalyGrade: toFixedNumberForAnomaly( get(rawAnomaly, 'anomaly_grade') @@ -584,7 +604,23 @@ export const parseBucketizedAnomalyResults = (result: any): Anomalies => { endTime: get(rawAnomaly, 'data_end_time'), plotTime: get(rawAnomaly, 'data_end_time'), entity: get(rawAnomaly, 'entity'), + contributions: toFixedNumberForAnomaly( + get(rawAnomaly, 'anomaly_grade') + ) > 0 ? contributions : {} + }); + + const featureBreakdownAttributionMap: {[key: string]: number} = {}; + + get(rawAnomaly, 'relevant_attribution', []).filter(element => get(element, 'feature_id') && get(element, 'data')). + forEach((attribution: any) => { + featureBreakdownAttributionMap[get(attribution, 'feature_id')] = toFixedNumberForAnomaly(get(attribution, 'data')); + }); + + const expectedValueMap: {[key: string]: number} = {}; + get(rawAnomaly, 'expected_values[0].value_list', []).forEach((expect: any) => { + expectedValueMap[get(expect, 'feature_id')] = toFixedNumberForAnomaly(get(expect, 'data')); }); + get(rawAnomaly, 'feature_data', []).forEach((feature) => { if (!get(featureData, get(feature, 'feature_id'))) { featureData[get(feature, 'feature_id')] = []; @@ -594,6 +630,8 @@ export const parseBucketizedAnomalyResults = (result: any): Anomalies => { startTime: get(rawAnomaly, 'data_start_time'), endTime: get(rawAnomaly, 'data_end_time'), plotTime: get(rawAnomaly, 'data_end_time'), + attribution: featureBreakdownAttributionMap[get(feature, 'feature_id')], + expectedValue: expectedValueMap[get(feature, 'feature_id')] }); }); } @@ -674,6 +712,23 @@ export const parsePureAnomalies = ( if (anomaliesHits.length > 0) { anomaliesHits.forEach((item: any) => { const rawAnomaly = get(item, '_source'); + const featureDataMap: {[key: string]: string} = {}; + get(rawAnomaly, 'feature_data', []).filter(element => get(element, 'feature_id') && get(element, 'feature_name')). + forEach((feature: any) => { + featureDataMap[get(feature, 'feature_id')] = get(feature, 'feature_name'); + }); + const relevantAttributionMap: {[key: string]: number} = {}; + get(rawAnomaly, 'relevant_attribution', []).filter(element => get(element, 'feature_id') && get(element, 'data')). + forEach((attribution: any) => { + relevantAttributionMap[get(attribution, 'feature_id')] = toFixedNumberForAnomaly(get(attribution, 'data')); + }); + const contributions: { [key: string]: FeatureContributionData } = {} + for (const [key] of Object.entries(featureDataMap)) { + contributions[key] = { + name: featureDataMap[key], + attribution: relevantAttributionMap[key] + } + } anomalies.push({ anomalyGrade: toFixedNumberForAnomaly(get(rawAnomaly, 'anomaly_grade')), confidence: toFixedNumberForAnomaly(get(rawAnomaly, 'confidence')), @@ -681,6 +736,7 @@ export const parsePureAnomalies = ( endTime: get(rawAnomaly, 'data_end_time'), plotTime: get(rawAnomaly, 'data_end_time'), entity: get(rawAnomaly, 'entity'), + contributions: contributions }); }); } diff --git a/server/models/types.ts b/server/models/types.ts index 42c51832..24fa7bb3 100644 --- a/server/models/types.ts +++ b/server/models/types.ts @@ -121,6 +121,12 @@ export type AnomalyResult = { confidence: number; entity?: Entity[]; features?: { [key: string]: FeatureResult }; + contributions?: { [key: string]: FeatureContributionData }; +}; + +export type FeatureContributionData = { + name: string; + attribution: number; }; export type FeatureResult = { @@ -128,6 +134,8 @@ export type FeatureResult = { endTime: number; plotTime: number; data: number; + name: string; + expectedValue?: number; }; export type AnomalyResultsResponse = { diff --git a/server/routes/ad.ts b/server/routes/ad.ts index bfdf2476..b5b345f3 100644 --- a/server/routes/ad.ts +++ b/server/routes/ad.ts @@ -963,11 +963,14 @@ export default class AdService { const detectorResult: AnomalyResult[] = []; const featureResult: { [key: string]: FeatureResult[] } = {}; + get(response, 'hits.hits', []).forEach((result: any) => { detectorResult.push({ startTime: result._source.data_start_time, endTime: result._source.data_end_time, plotTime: result._source.data_end_time, + contributions: result._source.anomaly_grade > 0 + ? result._source.relevant_attribution : {}, confidence: result._source.confidence != null && result._source.confidence !== 'NaN' && @@ -992,6 +995,7 @@ export default class AdService { // to know feature data belongs to which anomaly result features: this.getFeatureData(result), }); + result._source.feature_data.forEach((featureData: any) => { if (!featureResult[featureData.feature_id]) { featureResult[featureData.feature_id] = []; @@ -1004,6 +1008,9 @@ export default class AdService { featureData.data != null && featureData.data !== 'NaN' ? toFixedNumberForAnomaly(Number.parseFloat(featureData.data)) : 0, + name: featureData.feature_name, + expectedValue: result.anomaly_grade > 0 + ? this.getExpectedValue(result, featureData.feature_id) : featureData.data }); }); }); @@ -1127,8 +1134,23 @@ export default class AdService { featureData.data != null && featureData.data !== 'NaN' ? toFixedNumberForAnomaly(Number.parseFloat(featureData.data)) : 0, + name: featureData.feature_name, + expectedValue: rawResult.anomaly_grade > 0 + ? this.getExpectedValue(rawResult, featureData.feature_id) + : featureData.data }; }); return featureResult; }; + + getExpectedValue = (rawResult: any, featureId: string) => { + const expectedValueList = rawResult._source.expected_values; + if (expectedValueList.length > 0) { + expectedValueList[0].value_list.forEach((expect: any) => { + if (expect.feature_id === featureId) { + return expect.data; + } + }) + } + } }