1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define STATSD_DEBUG false
18
19 #include "Log.h"
20
21 #include "DurationMetricProducer.h"
22
23 #include <limits.h>
24 #include <stdlib.h>
25
26 #include "guardrail/StatsdStats.h"
27 #include "metrics/parsing_utils/metrics_manager_util.h"
28 #include "stats_log_util.h"
29 #include "stats_util.h"
30
31 using android::util::FIELD_COUNT_REPEATED;
32 using android::util::FIELD_TYPE_BOOL;
33 using android::util::FIELD_TYPE_FLOAT;
34 using android::util::FIELD_TYPE_INT32;
35 using android::util::FIELD_TYPE_INT64;
36 using android::util::FIELD_TYPE_MESSAGE;
37 using android::util::FIELD_TYPE_STRING;
38 using android::util::ProtoOutputStream;
39 using std::string;
40 using std::unordered_map;
41 using std::vector;
42 using std::shared_ptr;
43
44 namespace android {
45 namespace os {
46 namespace statsd {
47
48 // for StatsLogReport
49 const int FIELD_ID_ID = 1;
50 const int FIELD_ID_DURATION_METRICS = 6;
51 const int FIELD_ID_TIME_BASE = 9;
52 const int FIELD_ID_BUCKET_SIZE = 10;
53 const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
54 const int FIELD_ID_IS_ACTIVE = 14;
55 const int FIELD_ID_DIMENSION_GUARDRAIL_HIT = 17;
56 const int FIELD_ID_ESTIMATED_MEMORY_BYTES = 18;
57 const int FIELD_ID_DATA_CORRUPTED_REASON = 19;
58 // for DurationMetricDataWrapper
59 const int FIELD_ID_DATA = 1;
60 // for DurationMetricData
61 const int FIELD_ID_DIMENSION_IN_WHAT = 1;
62 const int FIELD_ID_BUCKET_INFO = 3;
63 const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
64 const int FIELD_ID_SLICE_BY_STATE = 6;
65 // for DurationBucketInfo
66 const int FIELD_ID_DURATION = 3;
67 const int FIELD_ID_BUCKET_NUM = 4;
68 const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5;
69 const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6;
70 const int FIELD_ID_CONDITION_TRUE_NS = 7;
71
DurationMetricProducer(const ConfigKey & key,const DurationMetric & metric,const int conditionIndex,const vector<ConditionState> & initialConditionCache,const int whatIndex,const int startIndex,const int stopIndex,const int stopAllIndex,const bool nesting,const sp<ConditionWizard> & wizard,const uint64_t protoHash,const FieldMatcher & internalDimensions,const int64_t timeBaseNs,const int64_t startTimeNs,const wp<ConfigMetadataProvider> configMetadataProvider,const unordered_map<int,shared_ptr<Activation>> & eventActivationMap,const unordered_map<int,vector<shared_ptr<Activation>>> & eventDeactivationMap,const vector<int> & slicedStateAtoms,const unordered_map<int,unordered_map<int,int64_t>> & stateGroupMap)72 DurationMetricProducer::DurationMetricProducer(
73 const ConfigKey& key, const DurationMetric& metric, const int conditionIndex,
74 const vector<ConditionState>& initialConditionCache, const int whatIndex,
75 const int startIndex, const int stopIndex, const int stopAllIndex, const bool nesting,
76 const sp<ConditionWizard>& wizard, const uint64_t protoHash,
77 const FieldMatcher& internalDimensions, const int64_t timeBaseNs, const int64_t startTimeNs,
78 const wp<ConfigMetadataProvider> configMetadataProvider,
79 const unordered_map<int, shared_ptr<Activation>>& eventActivationMap,
80 const unordered_map<int, vector<shared_ptr<Activation>>>& eventDeactivationMap,
81 const vector<int>& slicedStateAtoms,
82 const unordered_map<int, unordered_map<int, int64_t>>& stateGroupMap)
83 : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, initialConditionCache, wizard,
84 protoHash, eventActivationMap, eventDeactivationMap, slicedStateAtoms,
85 stateGroupMap, getAppUpgradeBucketSplit(metric), configMetadataProvider),
86 mAggregationType(metric.aggregation_type()),
87 mStartIndex(startIndex),
88 mStopIndex(stopIndex),
89 mStopAllIndex(stopAllIndex),
90 mNested(nesting),
91 mContainANYPositionInInternalDimensions(false),
92 mDimensionHardLimit(
93 StatsdStats::clampDimensionKeySizeLimit(metric.max_dimensions_per_bucket())) {
94 if (metric.has_bucket()) {
95 mBucketSizeNs =
96 TimeUnitToBucketSizeInMillisGuardrailed(key.GetUid(), metric.bucket()) * 1000000;
97 } else {
98 mBucketSizeNs = LLONG_MAX;
99 }
100
101 if (metric.has_threshold()) {
102 mUploadThreshold = metric.threshold();
103 }
104
105 if (metric.has_dimensions_in_what()) {
106 translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat);
107 mContainANYPositionInDimensionsInWhat = HasPositionANY(metric.dimensions_in_what());
108 }
109
110 if (internalDimensions.has_field()) {
111 translateFieldMatcher(internalDimensions, &mInternalDimensions);
112 mContainANYPositionInInternalDimensions = HasPositionANY(internalDimensions);
113 }
114 if (mContainANYPositionInInternalDimensions) {
115 ALOGE("Position ANY in internal dimension not supported.");
116 }
117 if (mContainANYPositionInDimensionsInWhat) {
118 ALOGE("Position ANY in dimension_in_what not supported.");
119 }
120
121 // Dimensions in what must be subset of internal dimensions
122 if (!subsetDimensions(mDimensionsInWhat, mInternalDimensions)) {
123 ALOGE("Dimensions in what must be a subset of the internal dimensions");
124 // TODO: Add invalidConfigReason
125 mValid = false;
126 }
127
128 mShouldUseNestedDimensions = ShouldUseNestedDimensions(metric.dimensions_in_what());
129
130 if (metric.links().size() > 0) {
131 for (const auto& link : metric.links()) {
132 Metric2Condition mc;
133 mc.conditionId = link.condition();
134 translateFieldMatcher(link.fields_in_what(), &mc.metricFields);
135 translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields);
136 if (!subsetDimensions(mc.metricFields, mInternalDimensions)) {
137 ALOGE(("Condition links must be a subset of the internal dimensions"));
138 // TODO: Add invalidConfigReason
139 mValid = false;
140 }
141 mMetric2ConditionLinks.push_back(mc);
142 }
143 mConditionSliced = true;
144 }
145 mUnSlicedPartCondition = ConditionState::kUnknown;
146
147 for (const auto& stateLink : metric.state_link()) {
148 Metric2State ms;
149 ms.stateAtomId = stateLink.state_atom_id();
150 translateFieldMatcher(stateLink.fields_in_what(), &ms.metricFields);
151 translateFieldMatcher(stateLink.fields_in_state(), &ms.stateFields);
152 if (!subsetDimensions(ms.metricFields, mInternalDimensions)) {
153 ALOGE(("State links must be a subset of the dimensions in what internal dimensions"));
154 // TODO: Add invalidConfigReason
155 mValid = false;
156 }
157 mMetric2StateLinks.push_back(ms);
158 }
159
160 mUseWhatDimensionAsInternalDimension = equalDimensions(mDimensionsInWhat, mInternalDimensions);
161 if (mWizard != nullptr && mConditionTrackerIndex >= 0 &&
162 mMetric2ConditionLinks.size() == 1) {
163 mHasLinksToAllConditionDimensionsInTracker = mWizard->equalOutputDimensions(
164 mConditionTrackerIndex, mMetric2ConditionLinks.begin()->conditionFields);
165 }
166 flushIfNeededLocked(startTimeNs);
167 // Adjust start for partial bucket
168 mCurrentBucketStartTimeNs = startTimeNs;
169 VLOG("metric %lld created. bucket size %lld start_time: %lld", (long long)mMetricId,
170 (long long)mBucketSizeNs, (long long)mTimeBaseNs);
171
172 initTrueDimensions(whatIndex, startTimeNs);
173 mConditionTimer.newBucketStart(mCurrentBucketStartTimeNs, mCurrentBucketStartTimeNs);
174 mConditionTimer.onConditionChanged(mIsActive && mCondition == ConditionState::kTrue,
175 mCurrentBucketStartTimeNs);
176 }
177
~DurationMetricProducer()178 DurationMetricProducer::~DurationMetricProducer() {
179 VLOG("~DurationMetric() called");
180 }
181
onConfigUpdatedLocked(const StatsdConfig & config,const int configIndex,const int metricIndex,const vector<sp<AtomMatchingTracker>> & allAtomMatchingTrackers,const unordered_map<int64_t,int> & oldAtomMatchingTrackerMap,const unordered_map<int64_t,int> & newAtomMatchingTrackerMap,const sp<EventMatcherWizard> & matcherWizard,const vector<sp<ConditionTracker>> & allConditionTrackers,const unordered_map<int64_t,int> & conditionTrackerMap,const sp<ConditionWizard> & wizard,const unordered_map<int64_t,int> & metricToActivationMap,unordered_map<int,vector<int>> & trackerToMetricMap,unordered_map<int,vector<int>> & conditionToMetricMap,unordered_map<int,vector<int>> & activationAtomTrackerToMetricMap,unordered_map<int,vector<int>> & deactivationAtomTrackerToMetricMap,vector<int> & metricsWithActivation)182 optional<InvalidConfigReason> DurationMetricProducer::onConfigUpdatedLocked(
183 const StatsdConfig& config, const int configIndex, const int metricIndex,
184 const vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers,
185 const unordered_map<int64_t, int>& oldAtomMatchingTrackerMap,
186 const unordered_map<int64_t, int>& newAtomMatchingTrackerMap,
187 const sp<EventMatcherWizard>& matcherWizard,
188 const vector<sp<ConditionTracker>>& allConditionTrackers,
189 const unordered_map<int64_t, int>& conditionTrackerMap, const sp<ConditionWizard>& wizard,
190 const unordered_map<int64_t, int>& metricToActivationMap,
191 unordered_map<int, vector<int>>& trackerToMetricMap,
192 unordered_map<int, vector<int>>& conditionToMetricMap,
193 unordered_map<int, vector<int>>& activationAtomTrackerToMetricMap,
194 unordered_map<int, vector<int>>& deactivationAtomTrackerToMetricMap,
195 vector<int>& metricsWithActivation) {
196 optional<InvalidConfigReason> invalidConfigReason = MetricProducer::onConfigUpdatedLocked(
197 config, configIndex, metricIndex, allAtomMatchingTrackers, oldAtomMatchingTrackerMap,
198 newAtomMatchingTrackerMap, matcherWizard, allConditionTrackers, conditionTrackerMap,
199 wizard, metricToActivationMap, trackerToMetricMap, conditionToMetricMap,
200 activationAtomTrackerToMetricMap, deactivationAtomTrackerToMetricMap,
201 metricsWithActivation);
202 if (invalidConfigReason.has_value()) {
203 return invalidConfigReason;
204 }
205
206 const DurationMetric& metric = config.duration_metric(configIndex);
207 const auto& what_it = conditionTrackerMap.find(metric.what());
208 if (what_it == conditionTrackerMap.end()) {
209 ALOGE("DurationMetric's \"what\" is not present in the config");
210 return createInvalidConfigReasonWithPredicate(
211 INVALID_CONFIG_REASON_DURATION_METRIC_WHAT_NOT_FOUND, mMetricId, metric.what());
212 }
213
214 const Predicate& durationWhat = config.predicate(what_it->second);
215 if (durationWhat.contents_case() != Predicate::ContentsCase::kSimplePredicate) {
216 ALOGE("DurationMetric's \"what\" must be a simple condition");
217 return createInvalidConfigReasonWithPredicate(
218 INVALID_CONFIG_REASON_DURATION_METRIC_WHAT_NOT_SIMPLE, mMetricId, metric.what());
219 }
220
221 const SimplePredicate& simplePredicate = durationWhat.simple_predicate();
222
223 // Update indices: mStartIndex, mStopIndex, mStopAllIndex, mConditionIndex and MetricsManager
224 // maps.
225 invalidConfigReason = handleMetricWithAtomMatchingTrackers(
226 simplePredicate.start(), mMetricId, metricIndex, metric.has_dimensions_in_what(),
227 allAtomMatchingTrackers, newAtomMatchingTrackerMap, trackerToMetricMap, mStartIndex);
228 if (invalidConfigReason.has_value()) {
229 ALOGE("Duration metrics must specify a valid start event matcher");
230 return invalidConfigReason;
231 }
232
233 if (simplePredicate.has_stop()) {
234 invalidConfigReason = handleMetricWithAtomMatchingTrackers(
235 simplePredicate.stop(), mMetricId, metricIndex, metric.has_dimensions_in_what(),
236 allAtomMatchingTrackers, newAtomMatchingTrackerMap, trackerToMetricMap, mStopIndex);
237 if (invalidConfigReason.has_value()) {
238 return invalidConfigReason;
239 }
240 }
241
242 if (simplePredicate.has_stop_all()) {
243 invalidConfigReason = handleMetricWithAtomMatchingTrackers(
244 simplePredicate.stop_all(), mMetricId, metricIndex, metric.has_dimensions_in_what(),
245 allAtomMatchingTrackers, newAtomMatchingTrackerMap, trackerToMetricMap,
246 mStopAllIndex);
247 if (invalidConfigReason.has_value()) {
248 return invalidConfigReason;
249 }
250 }
251
252 if (metric.has_condition()) {
253 invalidConfigReason = handleMetricWithConditions(
254 metric.condition(), mMetricId, metricIndex, conditionTrackerMap, metric.links(),
255 allConditionTrackers, mConditionTrackerIndex, conditionToMetricMap);
256 if (invalidConfigReason.has_value()) {
257 return invalidConfigReason;
258 }
259 }
260
261 for (const auto& it : mCurrentSlicedDurationTrackerMap) {
262 it.second->onConfigUpdated(wizard, mConditionTrackerIndex);
263 }
264
265 return nullopt;
266 }
267
initTrueDimensions(const int whatIndex,const int64_t startTimeNs)268 void DurationMetricProducer::initTrueDimensions(const int whatIndex, const int64_t startTimeNs) {
269 std::lock_guard<std::mutex> lock(mMutex);
270 // Currently whatIndex will only be -1 in tests. In the future, we might want to avoid creating
271 // a ConditionTracker if the condition is only used in the "what" of a duration metric. In that
272 // scenario, -1 can also be passed.
273 if (whatIndex == -1) {
274 return;
275 }
276 const map<HashableDimensionKey, int>* slicedWhatMap = mWizard->getSlicedDimensionMap(whatIndex);
277 for (const auto& [internalDimKey, count] : *slicedWhatMap) {
278 for (int i = 0; i < count; i++) {
279 // Fake start events.
280 handleMatchedLogEventValuesLocked(mStartIndex, internalDimKey.getValues(), startTimeNs);
281 }
282 }
283 }
284
addAnomalyTracker(const Alert & alert,const sp<AlarmMonitor> & anomalyAlarmMonitor,const UpdateStatus & updateStatus,const int64_t updateTimeNs)285 sp<AnomalyTracker> DurationMetricProducer::addAnomalyTracker(
286 const Alert& alert, const sp<AlarmMonitor>& anomalyAlarmMonitor,
287 const UpdateStatus& updateStatus, const int64_t updateTimeNs) {
288 std::lock_guard<std::mutex> lock(mMutex);
289 if (mAggregationType == DurationMetric_AggregationType_SUM) {
290 if (alert.trigger_if_sum_gt() > alert.num_buckets() * mBucketSizeNs) {
291 ALOGW("invalid alert for SUM: threshold (%f) > possible recordable value (%d x %lld)",
292 alert.trigger_if_sum_gt(), alert.num_buckets(), (long long)mBucketSizeNs);
293 return nullptr;
294 }
295 }
296 sp<AnomalyTracker> anomalyTracker =
297 new DurationAnomalyTracker(alert, mConfigKey, anomalyAlarmMonitor);
298 // The update status is either new or replaced.
299 addAnomalyTrackerLocked(anomalyTracker, updateStatus, updateTimeNs);
300 return anomalyTracker;
301 }
302
303 // Adds an AnomalyTracker that has already been created.
304 // Note: this gets called on config updates, and will only get called if the metric and the
305 // associated alert are preserved, which means the AnomalyTracker must be a DurationAnomalyTracker.
addAnomalyTracker(sp<AnomalyTracker> & anomalyTracker,const int64_t updateTimeNs)306 void DurationMetricProducer::addAnomalyTracker(sp<AnomalyTracker>& anomalyTracker,
307 const int64_t updateTimeNs) {
308 std::lock_guard<std::mutex> lock(mMutex);
309 addAnomalyTrackerLocked(anomalyTracker, UpdateStatus::UPDATE_PRESERVE, updateTimeNs);
310 }
311
addAnomalyTrackerLocked(sp<AnomalyTracker> & anomalyTracker,const UpdateStatus & updateStatus,const int64_t updateTimeNs)312 void DurationMetricProducer::addAnomalyTrackerLocked(sp<AnomalyTracker>& anomalyTracker,
313 const UpdateStatus& updateStatus,
314 const int64_t updateTimeNs) {
315 mAnomalyTrackers.push_back(anomalyTracker);
316 for (const auto& [_, durationTracker] : mCurrentSlicedDurationTrackerMap) {
317 durationTracker->addAnomalyTracker(anomalyTracker, updateStatus, updateTimeNs);
318 }
319 }
onStateChanged(const int64_t eventTimeNs,const int32_t atomId,const HashableDimensionKey & primaryKey,const FieldValue & oldState,const FieldValue & newState)320 void DurationMetricProducer::onStateChanged(const int64_t eventTimeNs, const int32_t atomId,
321 const HashableDimensionKey& primaryKey,
322 const FieldValue& oldState,
323 const FieldValue& newState) {
324 std::lock_guard<std::mutex> lock(mMutex);
325 // Check if this metric has a StateMap. If so, map the new state value to
326 // the correct state group id.
327 FieldValue newStateCopy = newState;
328 mapStateValue(atomId, &newStateCopy);
329
330 flushIfNeededLocked(eventTimeNs);
331
332 // Each duration tracker is mapped to a different whatKey (a set of values from the
333 // dimensionsInWhat fields). We notify all trackers iff the primaryKey field values from the
334 // state change event are a subset of the tracker's whatKey field values.
335 //
336 // Ex. For a duration metric dimensioned on uid and tag:
337 // DurationTracker1 whatKey = uid: 1001, tag: 1
338 // DurationTracker2 whatKey = uid: 1002, tag 1
339 //
340 // If the state change primaryKey = uid: 1001, we only notify DurationTracker1 of a state
341 // change.
342 for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
343 if (!containsLinkedStateValues(whatIt.first, primaryKey, mMetric2StateLinks, atomId)) {
344 continue;
345 }
346 whatIt.second->onStateChanged(eventTimeNs, atomId, newStateCopy);
347 }
348 }
349
createDurationTracker(const MetricDimensionKey & eventKey) const350 unique_ptr<DurationTracker> DurationMetricProducer::createDurationTracker(
351 const MetricDimensionKey& eventKey) const {
352 switch (mAggregationType) {
353 case DurationMetric_AggregationType_SUM:
354 return make_unique<OringDurationTracker>(
355 mConfigKey, mMetricId, eventKey, mWizard, mConditionTrackerIndex, mNested,
356 mCurrentBucketStartTimeNs, mCurrentBucketNum, mTimeBaseNs, mBucketSizeNs,
357 mConditionSliced, mHasLinksToAllConditionDimensionsInTracker, mAnomalyTrackers);
358 case DurationMetric_AggregationType_MAX_SPARSE:
359 return make_unique<MaxDurationTracker>(
360 mConfigKey, mMetricId, eventKey, mWizard, mConditionTrackerIndex, mNested,
361 mCurrentBucketStartTimeNs, mCurrentBucketNum, mTimeBaseNs, mBucketSizeNs,
362 mConditionSliced, mHasLinksToAllConditionDimensionsInTracker, mAnomalyTrackers);
363 }
364 }
365
366 // SlicedConditionChange optimization case 1:
367 // 1. If combination condition, logical operation is AND, only one sliced child predicate.
368 // 2. The links covers all dimension fields in the sliced child condition predicate.
onSlicedConditionMayChangeLocked_opt1(const int64_t eventTime)369 void DurationMetricProducer::onSlicedConditionMayChangeLocked_opt1(const int64_t eventTime) {
370 if (mMetric2ConditionLinks.size() != 1 ||
371 !mHasLinksToAllConditionDimensionsInTracker) {
372 return;
373 }
374
375 bool currentUnSlicedPartCondition = true;
376 if (!mWizard->IsSimpleCondition(mConditionTrackerIndex)) {
377 ConditionState unslicedPartState =
378 mWizard->getUnSlicedPartConditionState(mConditionTrackerIndex);
379 // When the unsliced part is still false, return directly.
380 if (mUnSlicedPartCondition == ConditionState::kFalse &&
381 unslicedPartState == ConditionState::kFalse) {
382 return;
383 }
384 mUnSlicedPartCondition = unslicedPartState;
385 currentUnSlicedPartCondition = mUnSlicedPartCondition > 0;
386 }
387
388 auto dimensionsChangedToTrue = mWizard->getChangedToTrueDimensions(mConditionTrackerIndex);
389 auto dimensionsChangedToFalse = mWizard->getChangedToFalseDimensions(mConditionTrackerIndex);
390
391 // The condition change is from the unsliced predicates.
392 // We need to find out the true dimensions from the sliced predicate and flip their condition
393 // state based on the new unsliced condition state.
394 if (dimensionsChangedToTrue == nullptr || dimensionsChangedToFalse == nullptr ||
395 (dimensionsChangedToTrue->empty() && dimensionsChangedToFalse->empty())) {
396 const map<HashableDimensionKey, int>* slicedConditionMap =
397 mWizard->getSlicedDimensionMap(mConditionTrackerIndex);
398 for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
399 HashableDimensionKey linkedConditionDimensionKey;
400 getDimensionForCondition(whatIt.first.getValues(), mMetric2ConditionLinks[0],
401 &linkedConditionDimensionKey);
402 const auto& slicedConditionIt = slicedConditionMap->find(linkedConditionDimensionKey);
403 if (slicedConditionIt != slicedConditionMap->end() && slicedConditionIt->second > 0) {
404 whatIt.second->onConditionChanged(currentUnSlicedPartCondition, eventTime);
405 }
406 }
407 } else {
408 // Handle the condition change from the sliced predicate.
409 if (currentUnSlicedPartCondition) {
410 for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
411 HashableDimensionKey linkedConditionDimensionKey;
412 getDimensionForCondition(whatIt.first.getValues(), mMetric2ConditionLinks[0],
413 &linkedConditionDimensionKey);
414 if (dimensionsChangedToTrue->find(linkedConditionDimensionKey) !=
415 dimensionsChangedToTrue->end()) {
416 whatIt.second->onConditionChanged(true, eventTime);
417 }
418 if (dimensionsChangedToFalse->find(linkedConditionDimensionKey) !=
419 dimensionsChangedToFalse->end()) {
420 whatIt.second->onConditionChanged(false, eventTime);
421 }
422 }
423 }
424 }
425 }
426
onSlicedConditionMayChangeInternalLocked(const int64_t eventTimeNs)427 void DurationMetricProducer::onSlicedConditionMayChangeInternalLocked(const int64_t eventTimeNs) {
428 bool changeDimTrackable = mWizard->IsChangedDimensionTrackable(mConditionTrackerIndex);
429 if (changeDimTrackable && mHasLinksToAllConditionDimensionsInTracker) {
430 onSlicedConditionMayChangeLocked_opt1(eventTimeNs);
431 return;
432 }
433
434 // Now for each of the on-going event, check if the condition has changed for them.
435 for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
436 whatIt.second->onSlicedConditionMayChange(eventTimeNs);
437 }
438 }
439
onSlicedConditionMayChangeLocked(bool overallCondition,const int64_t eventTime)440 void DurationMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondition,
441 const int64_t eventTime) {
442 VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId);
443
444 if (!mIsActive) {
445 return;
446 }
447
448 flushIfNeededLocked(eventTime);
449
450 if (!mConditionSliced) {
451 return;
452 }
453
454 onSlicedConditionMayChangeInternalLocked(eventTime);
455 }
456
onActiveStateChangedLocked(const int64_t eventTimeNs,const bool isActive)457 void DurationMetricProducer::onActiveStateChangedLocked(const int64_t eventTimeNs,
458 const bool isActive) {
459 MetricProducer::onActiveStateChangedLocked(eventTimeNs, isActive);
460
461 if (!mConditionSliced) {
462 if (ConditionState::kTrue != mCondition) {
463 return;
464 }
465
466 if (isActive) {
467 flushIfNeededLocked(eventTimeNs);
468 }
469
470 for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
471 whatIt.second->onConditionChanged(isActive, eventTimeNs);
472 }
473 mConditionTimer.onConditionChanged(isActive, eventTimeNs);
474 } else if (isActive) {
475 flushIfNeededLocked(eventTimeNs);
476 onSlicedConditionMayChangeInternalLocked(eventTimeNs);
477 } else { // mConditionSliced == true && !isActive
478 for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
479 whatIt.second->onConditionChanged(isActive, eventTimeNs);
480 }
481 }
482 }
483
onConditionChangedLocked(const bool conditionMet,const int64_t eventTime)484 void DurationMetricProducer::onConditionChangedLocked(const bool conditionMet,
485 const int64_t eventTime) {
486 VLOG("Metric %lld onConditionChanged", (long long)mMetricId);
487 mCondition = conditionMet ? ConditionState::kTrue : ConditionState::kFalse;
488
489 if (!mIsActive) {
490 return;
491 }
492
493 flushIfNeededLocked(eventTime);
494 for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
495 whatIt.second->onConditionChanged(conditionMet, eventTime);
496 }
497
498 mConditionTimer.onConditionChanged(mCondition, eventTime);
499 }
500
dropDataLocked(const int64_t dropTimeNs)501 void DurationMetricProducer::dropDataLocked(const int64_t dropTimeNs) {
502 flushIfNeededLocked(dropTimeNs);
503 StatsdStats::getInstance().noteBucketDropped(mMetricId);
504 mPastBuckets.clear();
505 resetDataCorruptionFlagsLocked();
506 }
507
clearPastBucketsLocked(const int64_t dumpTimeNs)508 void DurationMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) {
509 flushIfNeededLocked(dumpTimeNs);
510 resetDataCorruptionFlagsLocked();
511 mPastBuckets.clear();
512 }
513
onDumpReportLocked(const int64_t dumpTimeNs,const bool include_current_partial_bucket,const bool erase_data,const DumpLatency dumpLatency,std::set<string> * str_set,std::set<int32_t> & usedUids,ProtoOutputStream * protoOutput)514 void DurationMetricProducer::onDumpReportLocked(
515 const int64_t dumpTimeNs, const bool include_current_partial_bucket, const bool erase_data,
516 const DumpLatency dumpLatency, std::set<string>* str_set, std::set<int32_t>& usedUids,
517 ProtoOutputStream* protoOutput) {
518 if (include_current_partial_bucket) {
519 flushLocked(dumpTimeNs);
520 } else {
521 flushIfNeededLocked(dumpTimeNs);
522 }
523
524 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
525 protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_IS_ACTIVE, isActiveLocked());
526
527 // Data corrupted reason
528 writeDataCorruptedReasons(*protoOutput, FIELD_ID_DATA_CORRUPTED_REASON,
529 mDataCorruptedDueToQueueOverflow != DataCorruptionSeverity::kNone,
530 mDataCorruptedDueToSocketLoss != DataCorruptionSeverity::kNone);
531
532 if (mPastBuckets.empty()) {
533 VLOG(" Duration metric, empty return");
534 if (erase_data) {
535 resetDataCorruptionFlagsLocked();
536 }
537 return;
538 }
539
540 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ESTIMATED_MEMORY_BYTES,
541 (long long)byteSizeLocked());
542
543 if (StatsdStats::getInstance().hasHitDimensionGuardrail(mMetricId)) {
544 protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_DIMENSION_GUARDRAIL_HIT, true);
545 }
546
547 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
548 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
549
550 if (!mShouldUseNestedDimensions) {
551 if (!mDimensionsInWhat.empty()) {
552 uint64_t dimenPathToken = protoOutput->start(
553 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
554 writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
555 protoOutput->end(dimenPathToken);
556 }
557 }
558
559 uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DURATION_METRICS);
560
561 VLOG("Duration metric %lld dump report now...", (long long)mMetricId);
562
563 for (const auto& pair : mPastBuckets) {
564 const MetricDimensionKey& dimensionKey = pair.first;
565 VLOG(" dimension key %s", dimensionKey.toString().c_str());
566
567 uint64_t wrapperToken =
568 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
569
570 // First fill dimension.
571 if (mShouldUseNestedDimensions) {
572 uint64_t dimensionToken = protoOutput->start(
573 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
574 writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), mUidFields, str_set,
575 usedUids, protoOutput);
576 protoOutput->end(dimensionToken);
577 } else {
578 writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
579 FIELD_ID_DIMENSION_LEAF_IN_WHAT, mUidFields, str_set,
580 usedUids, protoOutput);
581 }
582 // Then fill slice_by_state.
583 for (auto state : dimensionKey.getStateValuesKey().getValues()) {
584 uint64_t stateToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
585 FIELD_ID_SLICE_BY_STATE);
586 writeStateToProto(state, protoOutput);
587 protoOutput->end(stateToken);
588 }
589 // Then fill bucket_info (DurationBucketInfo).
590 for (const auto& bucket : pair.second) {
591 uint64_t bucketInfoToken = protoOutput->start(
592 FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
593 if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
594 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
595 (long long)NanoToMillis(bucket.mBucketStartNs));
596 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
597 (long long)NanoToMillis(bucket.mBucketEndNs));
598 } else {
599 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
600 (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
601 }
602 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_DURATION, (long long)bucket.mDuration);
603
604 // We only write the condition timer value if the metric has a
605 // condition and isn't sliced by state or condition.
606 // TODO(b/268531762): Slice the condition timer by state and condition
607 if (mConditionTrackerIndex >= 0 && mSlicedStateAtoms.empty() && !mConditionSliced) {
608 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_CONDITION_TRUE_NS,
609 (long long)bucket.mConditionTrueNs);
610 }
611
612 protoOutput->end(bucketInfoToken);
613 VLOG("\t bucket [%lld - %lld] duration: %lld", (long long)bucket.mBucketStartNs,
614 (long long)bucket.mBucketEndNs, (long long)bucket.mDuration);
615 }
616
617 protoOutput->end(wrapperToken);
618 }
619
620 protoOutput->end(protoToken);
621 if (erase_data) {
622 mPastBuckets.clear();
623 resetDataCorruptionFlagsLocked();
624 }
625 }
626
flushIfNeededLocked(const int64_t eventTimeNs)627 void DurationMetricProducer::flushIfNeededLocked(const int64_t eventTimeNs) {
628 int64_t currentBucketEndTimeNs = getCurrentBucketEndTimeNs();
629
630 if (currentBucketEndTimeNs > eventTimeNs) {
631 return;
632 }
633 VLOG("flushing...........");
634 int numBucketsForward = 1 + (eventTimeNs - currentBucketEndTimeNs) / mBucketSizeNs;
635 int64_t nextBucketNs = currentBucketEndTimeNs + (numBucketsForward - 1) * mBucketSizeNs;
636 flushCurrentBucketLocked(eventTimeNs, nextBucketNs);
637
638 mCurrentBucketNum += numBucketsForward;
639 }
640
flushCurrentBucketLocked(const int64_t eventTimeNs,const int64_t nextBucketStartTimeNs)641 void DurationMetricProducer::flushCurrentBucketLocked(const int64_t eventTimeNs,
642 const int64_t nextBucketStartTimeNs) {
643 const auto [globalConditionTrueNs, globalConditionCorrectionNs] =
644 mConditionTimer.newBucketStart(eventTimeNs, nextBucketStartTimeNs);
645
646 for (auto whatIt = mCurrentSlicedDurationTrackerMap.begin();
647 whatIt != mCurrentSlicedDurationTrackerMap.end();) {
648 if (whatIt->second->flushCurrentBucket(eventTimeNs, mUploadThreshold, globalConditionTrueNs,
649 &mPastBuckets)) {
650 VLOG("erase bucket for key %s", whatIt->first.toString().c_str());
651 whatIt = mCurrentSlicedDurationTrackerMap.erase(whatIt);
652 } else {
653 ++whatIt;
654 }
655 }
656
657 StatsdStats::getInstance().noteBucketCount(mMetricId);
658 mCurrentBucketStartTimeNs = nextBucketStartTimeNs;
659 // Reset mHasHitGuardrail boolean since bucket was reset
660 mHasHitGuardrail = false;
661 }
662
dumpStatesLocked(int out,bool verbose) const663 void DurationMetricProducer::dumpStatesLocked(int out, bool verbose) const {
664 if (mCurrentSlicedDurationTrackerMap.size() == 0) {
665 return;
666 }
667
668 dprintf(out, "DurationMetric %lld dimension size %lu\n", (long long)mMetricId,
669 (unsigned long)mCurrentSlicedDurationTrackerMap.size());
670 if (verbose) {
671 for (const auto& whatIt : mCurrentSlicedDurationTrackerMap) {
672 dprintf(out, "\t(what)%s\n", whatIt.first.toString().c_str());
673 whatIt.second->dumpStates(out, verbose);
674 }
675 }
676 }
677
hitGuardRailLocked(const MetricDimensionKey & newKey) const678 bool DurationMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) const {
679 auto whatIt = mCurrentSlicedDurationTrackerMap.find(newKey.getDimensionKeyInWhat());
680 if (whatIt == mCurrentSlicedDurationTrackerMap.end()) {
681 // 1. Report the tuple count if the tuple count > soft limit
682 if (mCurrentSlicedDurationTrackerMap.size() >= StatsdStats::kDimensionKeySizeSoftLimit) {
683 size_t newTupleCount = mCurrentSlicedDurationTrackerMap.size() + 1;
684 StatsdStats::getInstance().noteMetricDimensionSize(
685 mConfigKey, mMetricId, newTupleCount);
686 // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
687 if (newTupleCount > mDimensionHardLimit) {
688 if (!mHasHitGuardrail) {
689 ALOGE("DurationMetric %lld dropping data for what dimension key %s",
690 (long long)mMetricId, newKey.getDimensionKeyInWhat().toString().c_str());
691 mHasHitGuardrail = true;
692 }
693 StatsdStats::getInstance().noteHardDimensionLimitReached(mMetricId);
694 return true;
695 }
696 }
697 }
698 return false;
699 }
700
handleStartEvent(const MetricDimensionKey & eventKey,const ConditionKey & conditionKeys,bool condition,const int64_t eventTimeNs,const vector<FieldValue> & eventValues)701 void DurationMetricProducer::handleStartEvent(const MetricDimensionKey& eventKey,
702 const ConditionKey& conditionKeys, bool condition,
703 const int64_t eventTimeNs,
704 const vector<FieldValue>& eventValues) {
705 const auto& whatKey = eventKey.getDimensionKeyInWhat();
706 auto whatIt = mCurrentSlicedDurationTrackerMap.find(whatKey);
707 if (whatIt == mCurrentSlicedDurationTrackerMap.end()) {
708 if (hitGuardRailLocked(eventKey)) {
709 return;
710 }
711 mCurrentSlicedDurationTrackerMap[whatKey] = createDurationTracker(eventKey);
712 }
713
714 auto it = mCurrentSlicedDurationTrackerMap.find(whatKey);
715 if (mUseWhatDimensionAsInternalDimension) {
716 it->second->noteStart(whatKey, condition, eventTimeNs, conditionKeys, mDimensionHardLimit);
717 return;
718 }
719
720 if (mInternalDimensions.empty()) {
721 it->second->noteStart(DEFAULT_DIMENSION_KEY, condition, eventTimeNs, conditionKeys,
722 mDimensionHardLimit);
723 } else {
724 HashableDimensionKey dimensionKey = DEFAULT_DIMENSION_KEY;
725 filterValues(mInternalDimensions, eventValues, &dimensionKey);
726 it->second->noteStart(dimensionKey, condition, eventTimeNs, conditionKeys,
727 mDimensionHardLimit);
728 }
729 }
730
onMatchedLogEventInternalLocked(const size_t matcherIndex,const MetricDimensionKey & eventKey,const ConditionKey & conditionKeys,bool condition,const LogEvent & event,const map<int,HashableDimensionKey> & statePrimaryKeys)731 void DurationMetricProducer::onMatchedLogEventInternalLocked(
732 const size_t matcherIndex, const MetricDimensionKey& eventKey,
733 const ConditionKey& conditionKeys, bool condition, const LogEvent& event,
734 const map<int, HashableDimensionKey>& statePrimaryKeys) {
735 ALOGW("Not used in duration tracker.");
736 }
737
onMatchedLogEventLocked(const size_t matcherIndex,const LogEvent & event)738 void DurationMetricProducer::onMatchedLogEventLocked(const size_t matcherIndex,
739 const LogEvent& event) {
740 handleMatchedLogEventValuesLocked(matcherIndex, event.getValues(),
741 event.GetElapsedTimestampNs());
742 }
743
handleMatchedLogEventValuesLocked(const size_t matcherIndex,const vector<FieldValue> & values,const int64_t eventTimeNs)744 void DurationMetricProducer::handleMatchedLogEventValuesLocked(const size_t matcherIndex,
745 const vector<FieldValue>& values,
746 const int64_t eventTimeNs) {
747 if (eventTimeNs < mTimeBaseNs) {
748 return;
749 }
750
751 if (mIsActive) {
752 flushIfNeededLocked(eventTimeNs);
753 }
754
755 // Handles Stopall events.
756 if ((int)matcherIndex == mStopAllIndex) {
757 for (auto whatIt = mCurrentSlicedDurationTrackerMap.begin();
758 whatIt != mCurrentSlicedDurationTrackerMap.end();) {
759 whatIt->second->noteStopAll(eventTimeNs);
760 if (!whatIt->second->hasAccumulatedDuration()) {
761 VLOG("erase bucket for key %s", whatIt->first.toString().c_str());
762 whatIt = mCurrentSlicedDurationTrackerMap.erase(whatIt);
763 } else {
764 whatIt++;
765 }
766 }
767 return;
768 }
769
770 if (!passesSampleCheckLocked(values)) {
771 return;
772 }
773
774 HashableDimensionKey dimensionInWhat = DEFAULT_DIMENSION_KEY;
775 if (!mDimensionsInWhat.empty()) {
776 filterValues(mDimensionsInWhat, values, &dimensionInWhat);
777 }
778
779 // Stores atom id to primary key pairs for each state atom that the metric is
780 // sliced by.
781 std::map<int, HashableDimensionKey> statePrimaryKeys;
782
783 // For states with primary fields, use MetricStateLinks to get the primary
784 // field values from the log event. These values will form a primary key
785 // that will be used to query StateTracker for the correct state value.
786 for (const auto& stateLink : mMetric2StateLinks) {
787 getDimensionForState(values, stateLink, &statePrimaryKeys[stateLink.stateAtomId]);
788 }
789
790 // For each sliced state, query StateTracker for the state value using
791 // either the primary key from the previous step or the DEFAULT_DIMENSION_KEY.
792 //
793 // Expected functionality: for any case where the MetricStateLinks are
794 // initialized incorrectly (ex. # of state links != # of primary fields, no
795 // links are provided for a state with primary fields, links are provided
796 // in the wrong order, etc.), StateTracker will simply return kStateUnknown
797 // when queried using an incorrect key.
798 HashableDimensionKey stateValuesKey = DEFAULT_DIMENSION_KEY;
799 for (auto atomId : mSlicedStateAtoms) {
800 FieldValue value;
801 if (statePrimaryKeys.find(atomId) != statePrimaryKeys.end()) {
802 // found a primary key for this state, query using the key
803 queryStateValue(atomId, statePrimaryKeys[atomId], &value);
804 } else {
805 // if no MetricStateLinks exist for this state atom,
806 // query using the default dimension key (empty HashableDimensionKey)
807 queryStateValue(atomId, DEFAULT_DIMENSION_KEY, &value);
808 }
809 mapStateValue(atomId, &value);
810 stateValuesKey.addValue(value);
811 }
812
813 // Handles Stop events.
814 if ((int)matcherIndex == mStopIndex) {
815 if (mUseWhatDimensionAsInternalDimension) {
816 auto whatIt = mCurrentSlicedDurationTrackerMap.find(dimensionInWhat);
817 if (whatIt != mCurrentSlicedDurationTrackerMap.end()) {
818 whatIt->second->noteStop(dimensionInWhat, eventTimeNs, false);
819 if (!whatIt->second->hasAccumulatedDuration()) {
820 VLOG("erase bucket for key %s", whatIt->first.toString().c_str());
821 mCurrentSlicedDurationTrackerMap.erase(whatIt);
822 }
823 }
824 return;
825 }
826
827 HashableDimensionKey internalDimensionKey = DEFAULT_DIMENSION_KEY;
828 if (!mInternalDimensions.empty()) {
829 filterValues(mInternalDimensions, values, &internalDimensionKey);
830 }
831
832 auto whatIt = mCurrentSlicedDurationTrackerMap.find(dimensionInWhat);
833 if (whatIt != mCurrentSlicedDurationTrackerMap.end()) {
834 whatIt->second->noteStop(internalDimensionKey, eventTimeNs, false);
835 if (!whatIt->second->hasAccumulatedDuration()) {
836 VLOG("erase bucket for key %s", whatIt->first.toString().c_str());
837 mCurrentSlicedDurationTrackerMap.erase(whatIt);
838 }
839 }
840 return;
841 }
842
843 bool condition;
844 ConditionKey conditionKey;
845 if (mConditionSliced) {
846 for (const auto& link : mMetric2ConditionLinks) {
847 getDimensionForCondition(values, link, &conditionKey[link.conditionId]);
848 }
849
850 auto conditionState =
851 mWizard->query(mConditionTrackerIndex, conditionKey,
852 !mHasLinksToAllConditionDimensionsInTracker);
853 condition = conditionState == ConditionState::kTrue;
854 } else {
855 // TODO: The unknown condition state is not handled here, we should fix it.
856 condition = mCondition == ConditionState::kTrue;
857 }
858
859 condition = condition && mIsActive;
860
861 handleStartEvent(MetricDimensionKey(dimensionInWhat, stateValuesKey), conditionKey, condition,
862 eventTimeNs, values);
863 }
864
865 // Estimate for the size of a DurationBucket.
computeBucketSizeLocked(const bool isFullBucket,const MetricDimensionKey & dimKey,const bool isFirstBucket) const866 size_t DurationMetricProducer::computeBucketSizeLocked(const bool isFullBucket,
867 const MetricDimensionKey& dimKey,
868 const bool isFirstBucket) const {
869 size_t bucketSize =
870 MetricProducer::computeBucketSizeLocked(isFullBucket, dimKey, isFirstBucket);
871
872 // Duration Value
873 bucketSize += sizeof(int64_t);
874
875 // ConditionTrueNanos
876 if (mConditionTrackerIndex >= 0 && mSlicedStateAtoms.empty() && !mConditionSliced) {
877 bucketSize += sizeof(int64_t);
878 }
879
880 return bucketSize;
881 }
882
byteSizeLocked() const883 size_t DurationMetricProducer::byteSizeLocked() const {
884 size_t totalSize = 0;
885 sp<ConfigMetadataProvider> configMetadataProvider = getConfigMetadataProvider();
886 if (configMetadataProvider != nullptr && configMetadataProvider->useV2SoftMemoryCalculation()) {
887 bool hasHitDimensionGuardrail =
888 StatsdStats::getInstance().hasHitDimensionGuardrail(mMetricId);
889 totalSize += computeOverheadSizeLocked(!mPastBuckets.empty(), hasHitDimensionGuardrail);
890 for (const auto& pair : mPastBuckets) {
891 bool isFirstBucket = true;
892 for (const auto& bucket : pair.second) {
893 bool isFullBucket = bucket.mBucketEndNs - bucket.mBucketStartNs >= mBucketSizeNs;
894 totalSize +=
895 computeBucketSizeLocked(isFullBucket, /*dimKey=*/pair.first, isFirstBucket);
896 isFirstBucket = false;
897 }
898 }
899 return totalSize;
900 }
901 for (const auto& pair : mPastBuckets) {
902 totalSize += pair.second.size() * kBucketSize;
903 }
904 return totalSize;
905 }
906
determineCorruptionSeverity(int32_t,DataCorruptedReason,LostAtomType atomType) const907 MetricProducer::DataCorruptionSeverity DurationMetricProducer::determineCorruptionSeverity(
908 int32_t /*atomId*/, DataCorruptedReason /*reason*/, LostAtomType atomType) const {
909 switch (atomType) {
910 case LostAtomType::kWhat:
911 // in case of loss stop/start/stopall event the error will be propagated
912 // to next bucket
913 return DataCorruptionSeverity::kUnrecoverable;
914 case LostAtomType::kCondition:
915 case LostAtomType::kState:
916 return DataCorruptionSeverity::kUnrecoverable;
917 };
918 return DataCorruptionSeverity::kNone;
919 };
920
921 } // namespace statsd
922 } // namespace os
923 } // namespace android
924