1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <gtest/gtest_prod.h>
20 
21 #include <optional>
22 
23 #include "ValueMetricProducer.h"
24 #include "metrics/NumericValue.h"
25 #include "src/stats_util.h"
26 
27 namespace android {
28 namespace os {
29 namespace statsd {
30 
31 using Bases = std::vector<NumericValue>;
32 class NumericValueMetricProducer : public ValueMetricProducer<NumericValue, Bases> {
33 public:
34     NumericValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric,
35                                const uint64_t protoHash, const PullOptions& pullOptions,
36                                const BucketOptions& bucketOptions, const WhatOptions& whatOptions,
37                                const ConditionOptions& conditionOptions,
38                                const StateOptions& stateOptions,
39                                const ActivationOptions& activationOptions,
40                                const GuardrailOptions& guardrailOptions,
41                                const wp<ConfigMetadataProvider> configMetadataProvider);
42 
43     // Process data pulled on bucket boundary.
44     void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& allData, PullResult pullResult,
45                       int64_t originalPullTimeNs) override;
46 
47     // Determine if metric needs to pull
isPullNeeded()48     bool isPullNeeded() const override {
49         std::lock_guard<std::mutex> lock(mMutex);
50         return mIsActive && (mCondition == ConditionState::kTrue);
51     }
52 
getMetricType()53     inline MetricType getMetricType() const override {
54         return METRIC_TYPE_VALUE;
55     }
56 
57 protected:
58 private:
59     void prepareFirstBucketLocked() override;
60 
getConditionIdForMetric(const StatsdConfig & config,const int configIndex)61     inline optional<int64_t> getConditionIdForMetric(const StatsdConfig& config,
62                                                      const int configIndex) const override {
63         const ValueMetric& metric = config.value_metric(configIndex);
64         return metric.has_condition() ? make_optional(metric.condition()) : nullopt;
65     }
66 
getWhatAtomMatcherIdForMetric(const StatsdConfig & config,const int configIndex)67     inline int64_t getWhatAtomMatcherIdForMetric(const StatsdConfig& config,
68                                                  const int configIndex) const override {
69         return config.value_metric(configIndex).what();
70     }
71 
getConditionLinksForMetric(const StatsdConfig & config,const int configIndex)72     inline ConditionLinks getConditionLinksForMetric(const StatsdConfig& config,
73                                                      const int configIndex) const override {
74         return config.value_metric(configIndex).links();
75     }
76 
77     void onActiveStateChangedInternalLocked(const int64_t eventTimeNs,
78                                             const bool isActive) override;
79 
80     // Only called when mIsActive and the event is NOT too late.
81     void onConditionChangedInternalLocked(const ConditionState oldCondition,
82                                           const ConditionState newCondition,
83                                           const int64_t eventTimeNs) override;
84 
aggregatedValueToString(const NumericValue & value)85     inline std::string aggregatedValueToString(const NumericValue& value) const override {
86         return value.toString();
87     }
88 
89     // Mark the data as invalid.
90     void invalidateCurrentBucket(const int64_t dropTimeNs, const BucketDropReason reason) override;
91 
92     // Reset diff base and mHasGlobalBase
93     void resetBase();
94 
95     // Calculate previous bucket end time based on current time.
96     int64_t calcPreviousBucketEndTime(const int64_t currentTimeNs);
97 
multipleBucketsSkipped(const int64_t numBucketsForward)98     inline bool multipleBucketsSkipped(const int64_t numBucketsForward) const override {
99         return numBucketsForward > 1 && (isPulled() || mUseDiff);
100     }
101 
102     // Process events retrieved from a pull.
103     void accumulateEvents(const std::vector<std::shared_ptr<LogEvent>>& allData,
104                           int64_t originalPullTimeNs, int64_t eventElapsedTimeNs);
105 
106     void closeCurrentBucket(const int64_t eventTimeNs,
107                             const int64_t nextBucketStartTimeNs) override;
108 
109     PastBucket<NumericValue> buildPartialBucket(int64_t bucketEndTime,
110                                                 std::vector<Interval>& intervals) override;
111 
112     bool valuePassesThreshold(const Interval& interval) const;
113 
114     NumericValue getFinalValue(const Interval& interval) const;
115 
116     void initNextSlicedBucket(int64_t nextBucketStartTimeNs) override;
117 
118     void appendToFullBucket(const bool isFullBucketReached);
119 
120     bool hitFullBucketGuardRailLocked(const MetricDimensionKey& newKey);
121 
canSkipLogEventLocked(const MetricDimensionKey & eventKey,const bool condition,int64_t eventTimeNs,const map<int,HashableDimensionKey> & statePrimaryKeys)122     inline bool canSkipLogEventLocked(
123             const MetricDimensionKey& eventKey, const bool condition, int64_t eventTimeNs,
124             const map<int, HashableDimensionKey>& statePrimaryKeys) const override {
125         // For pushed metrics, can only skip if condition is false.
126         // For pulled metrics, can only skip if metric is not diffed and condition is false or
127         // unknown.
128         return (!isPulled() && !condition) ||
129                (isPulled() && !mUseDiff && mCondition != ConditionState::kTrue);
130     }
131 
132     bool aggregateFields(const int64_t eventTimeNs, const MetricDimensionKey& eventKey,
133                          const LogEvent& event, std::vector<Interval>& intervals,
134                          Bases& bases) override;
135 
136     void pullAndMatchEventsLocked(const int64_t timestampNs) override;
137 
138     DumpProtoFields getDumpProtoFields() const override;
139 
140     void writePastBucketAggregateToProto(const int aggIndex, const NumericValue& value,
141                                          const int sampleSize,
142                                          ProtoOutputStream* const protoOutput) const override;
143 
144     // Internal function to calculate the current used bytes.
145     size_t byteSizeLocked() const override;
146 
147     void combineValueFields(pair<LogEvent, std::vector<int>>& eventValues, const LogEvent& newEvent,
148                             const std::vector<int>& newValueIndices) const;
149 
getAggregationTypeLocked(int index)150     ValueMetric::AggregationType getAggregationTypeLocked(int index) const {
151         return mAggregationTypes.size() == 1 ? mAggregationTypes[0] : mAggregationTypes[index];
152     }
153 
154     // Should only be called if there is at least one HISTOGRAM in mAggregationTypes
155     const std::optional<const BinStarts>& getBinStarts(int valueFieldIndex) const;
156 
157     size_t getAggregatedValueSize(const NumericValue& value) const override;
158 
hasAvgAggregationType(const vector<ValueMetric::AggregationType> aggregationTypes)159     bool hasAvgAggregationType(const vector<ValueMetric::AggregationType> aggregationTypes) const {
160         for (const int aggType : aggregationTypes) {
161             if (aggType == ValueMetric_AggregationType_AVG) {
162                 return true;
163             }
164         }
165         return false;
166     }
167 
168     DataCorruptionSeverity determineCorruptionSeverity(int32_t atomId, DataCorruptedReason reason,
169                                                        LostAtomType atomType) const override;
170 
171     const bool mUseAbsoluteValueOnReset;
172 
173     const std::vector<ValueMetric::AggregationType> mAggregationTypes;
174 
175     const bool mIncludeSampleSize;
176 
177     const bool mUseDiff;
178 
179     const ValueMetric::ValueDirection mValueDirection;
180 
181     const bool mSkipZeroDiffOutput;
182 
183     // If true, use a zero value as base to compute the diff.
184     // This is used for new keys which are present in the new data but was not
185     // present in the base data.
186     // The default base will only be used if we have a global base.
187     const bool mUseZeroDefaultBase;
188 
189     // For pulled metrics, this is always set to true whenever a pull succeeds.
190     // It is set to false when a pull fails, or upon condition change to false.
191     // This is used to decide if we have the right base data to compute the
192     // diff against.
193     bool mHasGlobalBase;
194 
195     const int64_t mMaxPullDelayNs;
196 
197     // Deduped value fields for matching.
198     const std::vector<Matcher> mDedupedFieldMatchers;
199 
200     // For anomaly detection.
201     std::unordered_map<MetricDimensionKey, int64_t> mCurrentFullBucket;
202 
203     const std::vector<std::optional<const BinStarts>> mBinStartsList;
204 
205     FRIEND_TEST(NumericValueMetricProducerTest, TestAnomalyDetection);
206     FRIEND_TEST(NumericValueMetricProducerTest, TestBaseSetOnConditionChange);
207     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundariesOnConditionChange);
208     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryNoCondition);
209     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryWithCondition);
210     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryWithCondition2);
211     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketInvalidIfGlobalBaseIsNotSet);
212     FRIEND_TEST(NumericValueMetricProducerTest, TestCalcPreviousBucketEndTime);
213     FRIEND_TEST(NumericValueMetricProducerTest, TestDataIsNotUpdatedWhenNoConditionChanged);
214     FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onBucketBoundary);
215     FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onConditionChanged);
216     FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onDataPulled);
217     FRIEND_TEST(NumericValueMetricProducerTest, TestEventsWithNonSlicedCondition);
218     FRIEND_TEST(NumericValueMetricProducerTest, TestFirstBucket);
219     FRIEND_TEST(NumericValueMetricProducerTest, TestLateOnDataPulledWithDiff);
220     FRIEND_TEST(NumericValueMetricProducerTest, TestLateOnDataPulledWithoutDiff);
221     FRIEND_TEST(NumericValueMetricProducerTest, TestPartialResetOnBucketBoundaries);
222     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryFalse);
223     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryTrue);
224     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withFailure);
225     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withMultipleConditionChanges);
226     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withoutCondition);
227     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsNoCondition);
228     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset);
229     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsTakeZeroOnReset);
230     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsWithFiltering);
231     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledWithAppUpgradeDisabled);
232     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateAvg);
233     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateMax);
234     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateMin);
235     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateSum);
236     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedEventsWithCondition);
237     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedEventsWithoutCondition);
238     FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullDelayExceeded);
239     FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange);
240     FRIEND_TEST(NumericValueMetricProducerTest,
241                 TestResetBaseOnPullFailAfterConditionChange_EndOfBucket);
242     FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullFailBeforeConditionChange);
243     FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullTooLate);
244     FRIEND_TEST(NumericValueMetricProducerTest, TestSampleSize);
245     FRIEND_TEST(NumericValueMetricProducerTest, TestSkipZeroDiffOutput);
246     FRIEND_TEST(NumericValueMetricProducerTest, TestSkipZeroDiffOutputMultiValue);
247     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedState);
248     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMap);
249     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithPrimaryField_WithDimensions);
250     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithCondition);
251     FRIEND_TEST(NumericValueMetricProducerTest, TestTrimUnusedDimensionKey);
252     FRIEND_TEST(NumericValueMetricProducerTest, TestUseZeroDefaultBase);
253     FRIEND_TEST(NumericValueMetricProducerTest, TestUseZeroDefaultBaseWithPullFailures);
254     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMultipleDimensions);
255     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMissingDataInStateChange);
256     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithDataMissingInConditionChange);
257     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMissingDataThenFlushBucket);
258     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithNoPullOnBucketBoundary);
259     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithConditionFalseMultipleBuckets);
260     FRIEND_TEST(NumericValueMetricProducerTest,
261                 TestSlicedStateWithMultipleDimensionsMissingDataInPull);
262     FRIEND_TEST(NumericValueMetricProducerTest, TestUploadThreshold);
263     FRIEND_TEST(NumericValueMetricProducerTest, TestMultipleAggTypesPulled);
264     FRIEND_TEST(NumericValueMetricProducerTest, TestMultipleAggTypesPushed);
265 
266     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenOneConditionFailed);
267     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenInitialPullFailed);
268     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenLastPullFailed);
269     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenGuardRailHit);
270     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop,
271                 TestInvalidBucketWhenDumpReportRequested);
272     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop,
273                 TestInvalidBucketWhenAccumulateEventWrongBucket);
274     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop,
275                 TestInvalidBucketWhenMultipleBucketsSkipped);
276 
277     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestBucketBoundariesOnPartialBucket);
278     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket,
279                 TestFullBucketResetWhenLastBucketInvalid);
280     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPartialBucketCreated);
281     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPushedEvents);
282     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPulledValue);
283     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPulledValueWhileConditionFalse);
284 
285     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
286                 TestAlarmLatePullWhileConditionTrue);
287     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
288                 TestAlarmLatePullWithConditionChanged);
289     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
290                 TestAlarmLatePullWhileConditionFalse);
291     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
292                 TestLatePullOnConditionChangeFalse);
293     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
294                 TestLatePullOnConditionChangeTrue);
295     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestAlarmLatePullNoCondition);
296     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
297                 TestAlarmLatePullNoConditionWithSkipped);
298     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
299                 TestThresholdNotDefinedNoUpload);
300     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestThresholdDefinedZero);
301     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
302                 TestThresholdUploadPassWhenEqual);
303     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
304                 TestThresholdUploadPassWhenGreater);
305     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestThresholdUploadSkip);
306     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestLateStateChangeSlicedAtoms);
307 
308     FRIEND_TEST(NumericValueMetricProducerTest, TestSubsetDimensions);
309 
310     FRIEND_TEST(ConfigUpdateTest, TestUpdateValueMetrics);
311 
312     FRIEND_TEST(MetricsManagerUtilDimLimitTest, TestDimLimit);
313 
314     FRIEND_TEST(ConfigUpdateDimLimitTest, TestDimLimit);
315 
316     FRIEND_TEST(ValueMetricE2eTest, TestInitWithMultipleAggTypes);
317     FRIEND_TEST(ValueMetricE2eTest, TestInitWithDefaultAggType);
318 
319     friend class NumericValueMetricProducerTestHelper;
320 };
321 
322 }  // namespace statsd
323 }  // namespace os
324 }  // namespace android
325