1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef METRIC_PRODUCER_H
18 #define METRIC_PRODUCER_H
19
20 #include <src/active_config_list.pb.h>
21 #include <src/guardrail/stats_log_enums.pb.h>
22 #include <utils/RefBase.h>
23
24 #include <unordered_map>
25
26 #include "HashableDimensionKey.h"
27 #include "anomaly/AnomalyTracker.h"
28 #include "condition/ConditionTimer.h"
29 #include "condition/ConditionWizard.h"
30 #include "config/ConfigKey.h"
31 #include "config/ConfigMetadataProvider.h"
32 #include "guardrail/StatsdStats.h"
33 #include "matchers/EventMatcherWizard.h"
34 #include "matchers/matcher_util.h"
35 #include "packages/PackageInfoListener.h"
36 #include "src/statsd_metadata.pb.h" // MetricMetadata
37 #include "state/StateListener.h"
38 #include "state/StateManager.h"
39 #include "utils/DbUtils.h"
40 #include "utils/ShardOffsetProvider.h"
41 #include "utils/api_tracing.h"
42
43 namespace android {
44 namespace os {
45 namespace statsd {
46
47 // If the metric has no activation requirement, it will be active once the metric producer is
48 // created.
49 // If the metric needs to be activated by atoms, the metric producer will start
50 // with kNotActive state, turn to kActive or kActiveOnBoot when the activation event arrives, become
51 // kNotActive when it reaches the duration limit (timebomb). If the activation event arrives again
52 // before or after it expires, the event producer will be re-activated and ttl will be reset.
53 enum ActivationState {
54 kNotActive = 0,
55 kActive = 1,
56 kActiveOnBoot = 2,
57 };
58
59 enum DumpLatency {
60 // In some cases, we only have a short time range to do the dump, e.g. statsd is being killed.
61 // We might be able to return all the data in this mode. For instance, pull metrics might need
62 // to be pulled when the current bucket is requested.
63 FAST = 1,
64 // In other cases, it is fine for a dump to take more than a few milliseconds, e.g. config
65 // updates.
66 NO_TIME_CONSTRAINTS = 2
67 };
68
69 enum MetricType {
70 METRIC_TYPE_EVENT = 1,
71 METRIC_TYPE_COUNT = 2,
72 METRIC_TYPE_DURATION = 3,
73 METRIC_TYPE_GAUGE = 4,
74 METRIC_TYPE_VALUE = 5,
75 METRIC_TYPE_KLL = 6,
76 };
77
78 struct Activation {
ActivationActivation79 Activation(const ActivationType& activationType, int64_t ttlNs)
80 : ttl_ns(ttlNs),
81 start_ns(0),
82 state(ActivationState::kNotActive),
83 activationType(activationType) {
84 }
85
86 const int64_t ttl_ns;
87 int64_t start_ns;
88 ActivationState state;
89 const ActivationType activationType;
90 };
91
92 struct DropEvent {
93 // Reason for dropping the bucket and/or marking the bucket invalid.
94 BucketDropReason reason;
95 // The timestamp of the drop event.
96 int64_t dropTimeNs;
97 };
98
99 struct SkippedBucket {
100 // Start time of the dropped bucket.
101 int64_t bucketStartTimeNs;
102 // End time of the dropped bucket.
103 int64_t bucketEndTimeNs;
104 // List of events that invalidated this bucket.
105 std::vector<DropEvent> dropEvents;
106
resetSkippedBucket107 void reset() {
108 bucketStartTimeNs = 0;
109 bucketEndTimeNs = 0;
110 dropEvents.clear();
111 }
112 };
113
114 struct SamplingInfo {
115 // Matchers for sampled fields. Currently only one sampled dimension is supported.
116 std::vector<Matcher> sampledWhatFields;
117
118 int shardCount = 0;
119 };
120
121 template <class T>
getAppUpgradeBucketSplit(const T & metric)122 optional<bool> getAppUpgradeBucketSplit(const T& metric) {
123 return metric.has_split_bucket_for_app_upgrade()
124 ? std::make_optional<bool>(metric.split_bucket_for_app_upgrade())
125 : std::nullopt;
126 }
127
128 // A MetricProducer is responsible for compute one single metric, creating stats log report, and
129 // writing the report to dropbox. MetricProducers should respond to package changes as required in
130 // PackageInfoListener, but if none of the metrics are slicing by package name, then the update can
131 // be a no-op.
132 class MetricProducer : public virtual RefBase, public virtual StateListener {
133 public:
134 MetricProducer(int64_t metricId, const ConfigKey& key, int64_t timeBaseNs,
135 const int conditionIndex, const vector<ConditionState>& initialConditionCache,
136 const sp<ConditionWizard>& wizard, const uint64_t protoHash,
137 const std::unordered_map<int, std::shared_ptr<Activation>>& eventActivationMap,
138 const std::unordered_map<int, std::vector<std::shared_ptr<Activation>>>&
139 eventDeactivationMap,
140 const vector<int>& slicedStateAtoms,
141 const unordered_map<int, unordered_map<int, int64_t>>& stateGroupMap,
142 const optional<bool> splitBucketForAppUpgrade,
143 const wp<ConfigMetadataProvider> configMetadataProvider);
144
~MetricProducer()145 virtual ~MetricProducer(){};
146
initialCondition(const int conditionIndex,const vector<ConditionState> & initialConditionCache)147 ConditionState initialCondition(const int conditionIndex,
148 const vector<ConditionState>& initialConditionCache) const {
149 return conditionIndex >= 0 ? initialConditionCache[conditionIndex] : ConditionState::kTrue;
150 }
151
152 // Update appropriate state on config updates. Primarily, all indices need to be updated.
153 // This metric and all of its dependencies are guaranteed to be preserved across the update.
154 // This function also updates several maps used by metricsManager.
155 // This function clears all anomaly trackers. All anomaly trackers need to be added again.
onConfigUpdated(const StatsdConfig & config,int configIndex,int metricIndex,const std::vector<sp<AtomMatchingTracker>> & allAtomMatchingTrackers,const std::unordered_map<int64_t,int> & oldAtomMatchingTrackerMap,const std::unordered_map<int64_t,int> & newAtomMatchingTrackerMap,const sp<EventMatcherWizard> & matcherWizard,const std::vector<sp<ConditionTracker>> & allConditionTrackers,const std::unordered_map<int64_t,int> & conditionTrackerMap,const sp<ConditionWizard> & wizard,const std::unordered_map<int64_t,int> & metricToActivationMap,std::unordered_map<int,std::vector<int>> & trackerToMetricMap,std::unordered_map<int,std::vector<int>> & conditionToMetricMap,std::unordered_map<int,std::vector<int>> & activationAtomTrackerToMetricMap,std::unordered_map<int,std::vector<int>> & deactivationAtomTrackerToMetricMap,std::vector<int> & metricsWithActivation)156 optional<InvalidConfigReason> onConfigUpdated(
157 const StatsdConfig& config, int configIndex, int metricIndex,
158 const std::vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers,
159 const std::unordered_map<int64_t, int>& oldAtomMatchingTrackerMap,
160 const std::unordered_map<int64_t, int>& newAtomMatchingTrackerMap,
161 const sp<EventMatcherWizard>& matcherWizard,
162 const std::vector<sp<ConditionTracker>>& allConditionTrackers,
163 const std::unordered_map<int64_t, int>& conditionTrackerMap,
164 const sp<ConditionWizard>& wizard,
165 const std::unordered_map<int64_t, int>& metricToActivationMap,
166 std::unordered_map<int, std::vector<int>>& trackerToMetricMap,
167 std::unordered_map<int, std::vector<int>>& conditionToMetricMap,
168 std::unordered_map<int, std::vector<int>>& activationAtomTrackerToMetricMap,
169 std::unordered_map<int, std::vector<int>>& deactivationAtomTrackerToMetricMap,
170 std::vector<int>& metricsWithActivation) {
171 std::lock_guard<std::mutex> lock(mMutex);
172 return onConfigUpdatedLocked(config, configIndex, metricIndex, allAtomMatchingTrackers,
173 oldAtomMatchingTrackerMap, newAtomMatchingTrackerMap,
174 matcherWizard, allConditionTrackers, conditionTrackerMap,
175 wizard, metricToActivationMap, trackerToMetricMap,
176 conditionToMetricMap, activationAtomTrackerToMetricMap,
177 deactivationAtomTrackerToMetricMap, metricsWithActivation);
178 };
179
180 /**
181 * Force a partial bucket split on app upgrade
182 */
notifyAppUpgrade(int64_t eventTimeNs)183 void notifyAppUpgrade(int64_t eventTimeNs) {
184 std::lock_guard<std::mutex> lock(mMutex);
185 const bool splitBucket =
186 mSplitBucketForAppUpgrade ? mSplitBucketForAppUpgrade.value() : false;
187 if (!splitBucket) {
188 return;
189 }
190 notifyAppUpgradeInternalLocked(eventTimeNs);
191 };
192
notifyAppRemoved(int64_t eventTimeNs)193 void notifyAppRemoved(int64_t eventTimeNs) {
194 // Force buckets to split on removal also.
195 notifyAppUpgrade(eventTimeNs);
196 };
197
198 /**
199 * Force a partial bucket split on boot complete.
200 */
onStatsdInitCompleted(int64_t eventTimeNs)201 virtual void onStatsdInitCompleted(int64_t eventTimeNs) {
202 ATRACE_CALL();
203 std::lock_guard<std::mutex> lock(mMutex);
204 flushLocked(eventTimeNs);
205 }
206
207 // Consume the parsed stats log entry that already matched the "what" of the metric.
onMatchedLogEvent(const size_t matcherIndex,const LogEvent & event)208 void onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) {
209 std::lock_guard<std::mutex> lock(mMutex);
210 onMatchedLogEventLocked(matcherIndex, event);
211 }
212
213 enum class LostAtomType {
214 kWhat = 0,
215 kCondition,
216 kState,
217 };
218
onMatchedLogEventLost(int32_t atomId,DataCorruptedReason reason,LostAtomType atomType)219 void onMatchedLogEventLost(int32_t atomId, DataCorruptedReason reason, LostAtomType atomType) {
220 std::lock_guard<std::mutex> lock(mMutex);
221 onMatchedLogEventLostLocked(atomId, reason, atomType);
222 }
223
onConditionChanged(const bool condition,int64_t eventTime)224 void onConditionChanged(const bool condition, int64_t eventTime) {
225 std::lock_guard<std::mutex> lock(mMutex);
226 onConditionChangedLocked(condition, eventTime);
227 }
228
onSlicedConditionMayChange(bool overallCondition,int64_t eventTime)229 void onSlicedConditionMayChange(bool overallCondition, int64_t eventTime) {
230 std::lock_guard<std::mutex> lock(mMutex);
231 onSlicedConditionMayChangeLocked(overallCondition, eventTime);
232 }
233
isConditionSliced()234 bool isConditionSliced() const {
235 std::lock_guard<std::mutex> lock(mMutex);
236 return mConditionSliced;
237 };
238
onStateChanged(const int64_t eventTimeNs,const int32_t atomId,const HashableDimensionKey & primaryKey,const FieldValue & oldState,const FieldValue & newState)239 void onStateChanged(const int64_t eventTimeNs, const int32_t atomId,
240 const HashableDimensionKey& primaryKey, const FieldValue& oldState,
241 const FieldValue& newState){};
242
onStateEventLost(int32_t atomId,DataCorruptedReason reason)243 void onStateEventLost(int32_t atomId, DataCorruptedReason reason) override {
244 std::lock_guard<std::mutex> lock(mMutex);
245 onMatchedLogEventLostLocked(atomId, reason, LostAtomType::kState);
246 }
247
248 // Output the metrics data to [protoOutput]. All metrics reports end with the same timestamp.
249 // This method clears all the past buckets.
onDumpReport(const int64_t dumpTimeNs,const bool include_current_partial_bucket,const bool erase_data,const DumpLatency dumpLatency,std::set<string> * str_set,std::set<int32_t> & usedUids,android::util::ProtoOutputStream * protoOutput)250 void onDumpReport(const int64_t dumpTimeNs, const bool include_current_partial_bucket,
251 const bool erase_data, const DumpLatency dumpLatency,
252 std::set<string>* str_set, std::set<int32_t>& usedUids,
253 android::util::ProtoOutputStream* protoOutput) {
254 std::lock_guard<std::mutex> lock(mMutex);
255 onDumpReportLocked(dumpTimeNs, include_current_partial_bucket, erase_data, dumpLatency,
256 str_set, usedUids, protoOutput);
257 }
258
259 virtual optional<InvalidConfigReason> onConfigUpdatedLocked(
260 const StatsdConfig& config, int configIndex, int metricIndex,
261 const std::vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers,
262 const std::unordered_map<int64_t, int>& oldAtomMatchingTrackerMap,
263 const std::unordered_map<int64_t, int>& newAtomMatchingTrackerMap,
264 const sp<EventMatcherWizard>& matcherWizard,
265 const std::vector<sp<ConditionTracker>>& allConditionTrackers,
266 const std::unordered_map<int64_t, int>& conditionTrackerMap,
267 const sp<ConditionWizard>& wizard,
268 const std::unordered_map<int64_t, int>& metricToActivationMap,
269 std::unordered_map<int, std::vector<int>>& trackerToMetricMap,
270 std::unordered_map<int, std::vector<int>>& conditionToMetricMap,
271 std::unordered_map<int, std::vector<int>>& activationAtomTrackerToMetricMap,
272 std::unordered_map<int, std::vector<int>>& deactivationAtomTrackerToMetricMap,
273 std::vector<int>& metricsWithActivation);
274
clearPastBuckets(const int64_t dumpTimeNs)275 void clearPastBuckets(const int64_t dumpTimeNs) {
276 std::lock_guard<std::mutex> lock(mMutex);
277 clearPastBucketsLocked(dumpTimeNs);
278 }
279
prepareFirstBucket()280 void prepareFirstBucket() {
281 std::lock_guard<std::mutex> lock(mMutex);
282 prepareFirstBucketLocked();
283 }
284
285 // Returns the memory in bytes currently used to store this metric's data. Does not change
286 // state.
byteSize()287 size_t byteSize() const {
288 std::lock_guard<std::mutex> lock(mMutex);
289 return byteSizeLocked();
290 }
291
dumpStates(int out,bool verbose)292 void dumpStates(int out, bool verbose) const {
293 std::lock_guard<std::mutex> lock(mMutex);
294 dumpStatesLocked(out, verbose);
295 }
296
297 // Let MetricProducer drop in-memory data to save memory.
298 // We still need to keep future data valid and anomaly tracking work, which means we will
299 // have to flush old data, informing anomaly trackers then safely drop old data.
300 // We still keep current bucket data for future metrics' validity.
dropData(const int64_t dropTimeNs)301 void dropData(const int64_t dropTimeNs) {
302 std::lock_guard<std::mutex> lock(mMutex);
303 dropDataLocked(dropTimeNs);
304 }
305
loadActiveMetric(const ActiveMetric & activeMetric,int64_t currentTimeNs)306 void loadActiveMetric(const ActiveMetric& activeMetric, int64_t currentTimeNs) {
307 std::lock_guard<std::mutex> lock(mMutex);
308 loadActiveMetricLocked(activeMetric, currentTimeNs);
309 }
310
activate(int activationTrackerIndex,int64_t elapsedTimestampNs)311 void activate(int activationTrackerIndex, int64_t elapsedTimestampNs) {
312 std::lock_guard<std::mutex> lock(mMutex);
313 activateLocked(activationTrackerIndex, elapsedTimestampNs);
314 }
315
cancelEventActivation(int deactivationTrackerIndex)316 void cancelEventActivation(int deactivationTrackerIndex) {
317 std::lock_guard<std::mutex> lock(mMutex);
318 cancelEventActivationLocked(deactivationTrackerIndex);
319 }
320
isActive()321 bool isActive() const {
322 std::lock_guard<std::mutex> lock(mMutex);
323 return isActiveLocked();
324 }
325
326 void flushIfExpire(int64_t elapsedTimestampNs);
327
328 void writeActiveMetricToProtoOutputStream(int64_t currentTimeNs, const DumpReportReason reason,
329 ProtoOutputStream* proto);
330
enforceRestrictedDataTtl(sqlite3 * db,int64_t wallClockNs)331 virtual void enforceRestrictedDataTtl(sqlite3* db, int64_t wallClockNs){};
332
writeMetricMetadataToProto(metadata::MetricMetadata * metricMetadata)333 virtual bool writeMetricMetadataToProto(metadata::MetricMetadata* metricMetadata) {
334 return false;
335 }
336
loadMetricMetadataFromProto(const metadata::MetricMetadata & metricMetadata)337 virtual void loadMetricMetadataFromProto(const metadata::MetricMetadata& metricMetadata){};
338
339 /* Called when the metric is to about to be removed from config. */
onMetricRemove()340 virtual void onMetricRemove() {
341 }
342
flushRestrictedData()343 virtual void flushRestrictedData() {
344 }
345
346 // Start: getters/setters
getMetricId()347 inline int64_t getMetricId() const {
348 return mMetricId;
349 }
350
getProtoHash()351 inline uint64_t getProtoHash() const {
352 return mProtoHash;
353 }
354
355 virtual MetricType getMetricType() const = 0;
356
357 // For test only.
getCurrentBucketNum()358 inline int64_t getCurrentBucketNum() const {
359 return mCurrentBucketNum;
360 }
361
getSlicedStateAtoms()362 inline const std::vector<int> getSlicedStateAtoms() {
363 std::lock_guard<std::mutex> lock(mMutex);
364 return mSlicedStateAtoms;
365 }
366
isValid()367 inline bool isValid() const {
368 return mValid;
369 }
370
371 /* Adds an AnomalyTracker and returns it. */
addAnomalyTracker(const Alert & alert,const sp<AlarmMonitor> & anomalyAlarmMonitor,const UpdateStatus & updateStatus,const int64_t updateTimeNs)372 virtual sp<AnomalyTracker> addAnomalyTracker(const Alert& alert,
373 const sp<AlarmMonitor>& anomalyAlarmMonitor,
374 const UpdateStatus& updateStatus,
375 const int64_t updateTimeNs) {
376 std::lock_guard<std::mutex> lock(mMutex);
377 sp<AnomalyTracker> anomalyTracker = new AnomalyTracker(alert, mConfigKey);
378 mAnomalyTrackers.push_back(anomalyTracker);
379 return anomalyTracker;
380 }
381
382 /* Adds an AnomalyTracker that has already been created */
addAnomalyTracker(sp<AnomalyTracker> & anomalyTracker,int64_t updateTimeNs)383 virtual void addAnomalyTracker(sp<AnomalyTracker>& anomalyTracker, int64_t updateTimeNs) {
384 std::lock_guard<std::mutex> lock(mMutex);
385 mAnomalyTrackers.push_back(anomalyTracker);
386 }
387
setSamplingInfo(SamplingInfo samplingInfo)388 void setSamplingInfo(SamplingInfo samplingInfo) {
389 std::lock_guard<std::mutex> lock(mMutex);
390 mSampledWhatFields.swap(samplingInfo.sampledWhatFields);
391 mShardCount = samplingInfo.shardCount;
392 }
393
setUidFields(std::vector<Matcher> uidFields)394 void setUidFields(std::vector<Matcher> uidFields) {
395 std::lock_guard<std::mutex> lock(mMutex);
396 mUidFields.swap(uidFields);
397 }
398
399 // End: getters/setters
400 protected:
401 /**
402 * Flushes the current bucket if the eventTime is after the current bucket's end time.
403 */
flushIfNeededLocked(int64_t eventTime)404 virtual void flushIfNeededLocked(int64_t eventTime){};
405
406 /**
407 * For metrics that aggregate (ie, every metric producer except for EventMetricProducer),
408 * we need to be able to flush the current buckets on demand (ie, end the current bucket and
409 * start new bucket). If this function is called when eventTimeNs is greater than the current
410 * bucket's end timestamp, than we flush up to the end of the latest full bucket; otherwise,
411 * we assume that we want to flush a partial bucket. The bucket start timestamp and bucket
412 * number are not changed by this function. This method should only be called by
413 * flushIfNeededLocked or flushLocked or the app upgrade handler; the caller MUST update the
414 * bucket timestamp and bucket number as needed.
415 */
flushCurrentBucketLocked(int64_t eventTimeNs,int64_t nextBucketStartTimeNs)416 virtual void flushCurrentBucketLocked(int64_t eventTimeNs, int64_t nextBucketStartTimeNs){};
417
418 /**
419 * Flushes all the data including the current partial bucket.
420 */
flushLocked(int64_t eventTimeNs)421 void flushLocked(int64_t eventTimeNs) {
422 flushIfNeededLocked(eventTimeNs);
423 flushCurrentBucketLocked(eventTimeNs, eventTimeNs);
424 };
425
notifyAppUpgradeInternalLocked(const int64_t eventTimeNs)426 virtual void notifyAppUpgradeInternalLocked(const int64_t eventTimeNs) {
427 flushLocked(eventTimeNs);
428 }
429
430 /*
431 * Individual metrics can implement their own business logic here. All pre-processing is done.
432 *
433 * [matcherIndex]: the index of the matcher which matched this event. This is interesting to
434 * DurationMetric, because it has start/stop/stop_all 3 matchers.
435 * [eventKey]: the extracted dimension key for the final output. if the metric doesn't have
436 * dimensions, it will be DEFAULT_DIMENSION_KEY
437 * [conditionKey]: the keys of conditions which should be used to query the condition for this
438 * target event (from MetricConditionLink). This is passed to individual metrics
439 * because DurationMetric needs it to be cached.
440 * [condition]: whether condition is met. If condition is sliced, this is the result coming from
441 * query with ConditionWizard; If condition is not sliced, this is the
442 * nonSlicedCondition.
443 * [event]: the log event, just in case the metric needs its data, e.g., EventMetric.
444 */
445 virtual void onMatchedLogEventInternalLocked(
446 const size_t matcherIndex, const MetricDimensionKey& eventKey,
447 const ConditionKey& conditionKey, bool condition, const LogEvent& event,
448 const map<int, HashableDimensionKey>& statePrimaryKeys) = 0;
449
450 // Consume the parsed stats log entry that already matched the "what" of the metric.
451 virtual void onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event);
452 virtual void onMatchedLogEventLostLocked(int32_t atomId, DataCorruptedReason reason,
453 LostAtomType atomType);
454 virtual void onConditionChangedLocked(const bool condition, int64_t eventTime) = 0;
455 virtual void onSlicedConditionMayChangeLocked(bool overallCondition,
456 const int64_t eventTime) = 0;
457 virtual void onDumpReportLocked(const int64_t dumpTimeNs,
458 const bool include_current_partial_bucket,
459 const bool erase_data, const DumpLatency dumpLatency,
460 std::set<string>* str_set, std::set<int32_t>& usedUids,
461 android::util::ProtoOutputStream* protoOutput) = 0;
462 virtual void clearPastBucketsLocked(const int64_t dumpTimeNs) = 0;
prepareFirstBucketLocked()463 virtual void prepareFirstBucketLocked(){};
464 virtual size_t byteSizeLocked() const = 0;
465 virtual void dumpStatesLocked(int out, bool verbose) const = 0;
466 virtual void dropDataLocked(const int64_t dropTimeNs) = 0;
467 void loadActiveMetricLocked(const ActiveMetric& activeMetric, int64_t currentTimeNs);
468 void activateLocked(int activationTrackerIndex, int64_t elapsedTimestampNs);
469 void cancelEventActivationLocked(int deactivationTrackerIndex);
470
471 // Computes the size of a newly added bucket to this metric, taking into account any new
472 // dimensions that are introduced if necessary.
473 virtual size_t computeBucketSizeLocked(const bool isFullBucket,
474 const MetricDimensionKey& dimKey,
475 const bool isFirstBucket) const;
476 size_t computeOverheadSizeLocked(const bool hasPastBuckets,
477 const bool dimensionGuardrailHit) const;
478 size_t computeSkippedBucketSizeLocked(const SkippedBucket& skippedBucket) const;
479
480 bool evaluateActiveStateLocked(int64_t elapsedTimestampNs);
481
onActiveStateChangedLocked(const int64_t eventTimeNs,const bool isActive)482 virtual void onActiveStateChangedLocked(const int64_t eventTimeNs, const bool isActive) {
483 if (!isActive) {
484 flushLocked(eventTimeNs);
485 }
486 }
487
isActiveLocked()488 inline bool isActiveLocked() const {
489 return mIsActive;
490 }
491
492 // Convenience to compute the current bucket's end time, which is always aligned with the
493 // start time of the metric.
getCurrentBucketEndTimeNs()494 int64_t getCurrentBucketEndTimeNs() const {
495 return mTimeBaseNs + (mCurrentBucketNum + 1) * mBucketSizeNs;
496 }
497
getBucketNumFromEndTimeNs(const int64_t endNs)498 int64_t getBucketNumFromEndTimeNs(const int64_t endNs) {
499 return (endNs - mTimeBaseNs) / mBucketSizeNs - 1;
500 }
501
502 // Query StateManager for original state value using the queryKey.
503 // The field and value are output.
504 void queryStateValue(int32_t atomId, const HashableDimensionKey& queryKey, FieldValue* value);
505
506 // If a state map exists for the given atom, replace the original state
507 // value with the group id mapped to the value.
508 // If no state map exists, keep the original state value.
509 void mapStateValue(int32_t atomId, FieldValue* value);
510
511 // Returns a HashableDimensionKey with unknown state value for each state
512 // atom.
513 HashableDimensionKey getUnknownStateKey();
514
515 DropEvent buildDropEvent(const int64_t dropTimeNs, const BucketDropReason reason) const;
516
517 // Returns true if the number of drop events in the current bucket has
518 // exceeded the maximum number allowed, which is currently capped at 10.
519 bool maxDropEventsReached() const;
520
521 bool passesSampleCheckLocked(const vector<FieldValue>& values) const;
522
523 const int64_t mMetricId;
524
525 // Hash of the Metric's proto bytes from StatsdConfig, including any activations.
526 // Used to determine if the definition of this metric has changed across a config update.
527 const uint64_t mProtoHash;
528
529 const ConfigKey mConfigKey;
530
531 bool mValid;
532
533 // The time when this metric producer was first created. The end time for the current bucket
534 // can be computed from this based on mCurrentBucketNum.
535 const int64_t mTimeBaseNs;
536
537 // Start time may not be aligned with the start of statsd if there is an app upgrade in the
538 // middle of a bucket.
539 int64_t mCurrentBucketStartTimeNs;
540
541 // Used by anomaly detector to track which bucket we are in. This is not sent with the produced
542 // report.
543 int64_t mCurrentBucketNum;
544
545 int64_t mBucketSizeNs;
546
547 ConditionState mCondition;
548
549 ConditionTimer mConditionTimer;
550
551 int mConditionTrackerIndex;
552
553 // TODO(b/185770739): use !mMetric2ConditionLinks.empty()
554 bool mConditionSliced;
555
556 sp<ConditionWizard> mWizard;
557
558 bool mContainANYPositionInDimensionsInWhat;
559
560 // Metrics slicing by primitive repeated field and/or position ALL need to use nested
561 // dimensions.
562 bool mShouldUseNestedDimensions;
563
564 vector<Matcher> mDimensionsInWhat; // The dimensions_in_what defined in statsd_config
565
566 // True iff the metric to condition links cover all dimension fields in the condition tracker.
567 // This field is always false for combinational condition trackers.
568 bool mHasLinksToAllConditionDimensionsInTracker;
569
570 std::vector<Metric2Condition> mMetric2ConditionLinks;
571
572 std::vector<sp<AnomalyTracker>> mAnomalyTrackers;
573
574 mutable std::mutex mMutex;
575
576 // When the metric producer has multiple activations, these activations are ORed to determine
577 // whether the metric producer is ready to generate metrics.
578 std::unordered_map<int, std::shared_ptr<Activation>> mEventActivationMap;
579
580 // Maps index of atom matcher for deactivation to a list of Activation structs.
581 std::unordered_map<int, std::vector<std::shared_ptr<Activation>>> mEventDeactivationMap;
582
583 bool mIsActive;
584
585 // The slice_by_state atom ids defined in statsd_config.
586 const std::vector<int32_t> mSlicedStateAtoms;
587
588 // Maps atom ids and state values to group_ids (<atom_id, <value, group_id>>).
589 const std::unordered_map<int32_t, std::unordered_map<int, int64_t>> mStateGroupMap;
590
591 // MetricStateLinks defined in statsd_config that link fields in the state
592 // atom to fields in the "what" atom.
593 std::vector<Metric2State> mMetric2StateLinks;
594
595 optional<UploadThreshold> mUploadThreshold;
596
597 const optional<bool> mSplitBucketForAppUpgrade;
598
599 SkippedBucket mCurrentSkippedBucket;
600 // Buckets that were invalidated and had their data dropped.
601 std::vector<SkippedBucket> mSkippedBuckets;
602
603 // If hard dimension guardrail is hit, do not spam logcat. This is a per bucket tracker.
604 mutable bool mHasHitGuardrail;
605
606 // Matchers for sampled fields. Currently only one sampled dimension is supported.
607 std::vector<Matcher> mSampledWhatFields;
608
609 int mShardCount;
610
611 // For tracking uid fields in a metric. Only needed if the field is not annotated in the atom
612 // and omit_unused_uids_in_uidmap = true.
613 std::vector<Matcher> mUidFields;
614
615 sp<ConfigMetadataProvider> getConfigMetadataProvider() const;
616
617 wp<ConfigMetadataProvider> mConfigMetadataProvider;
618
619 enum DataCorruptionSeverity { kNone = 0, kResetOnDump, kUnrecoverable };
620
621 DataCorruptionSeverity mDataCorruptedDueToSocketLoss = DataCorruptionSeverity::kNone;
622 DataCorruptionSeverity mDataCorruptedDueToQueueOverflow = DataCorruptionSeverity::kNone;
623
624 /**
625 * @brief Determines DataCorruptionSeverity based on source and atom type being lost
626 *
627 * @return DataCorruptionSeverity
628 */
determineCorruptionSeverity(int32_t,DataCorruptedReason,LostAtomType)629 virtual DataCorruptionSeverity determineCorruptionSeverity(int32_t /*atomId*/,
630 DataCorruptedReason /*reason*/,
631 LostAtomType /*atomType*/) const {
632 return DataCorruptionSeverity::kNone;
633 };
634
635 /**
636 * @brief Resets data corruption reason info if no unrecoverable errors observed
637 */
638 void resetDataCorruptionFlagsLocked();
639
640 size_t mTotalDataSize = 0;
641
642 friend class SocketLossInfoTest;
643 FRIEND_TEST(CountMetricE2eTest, TestSlicedState);
644 FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithMap);
645 FRIEND_TEST(CountMetricE2eTest, TestMultipleSlicedStates);
646 FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithPrimaryFields);
647 FRIEND_TEST(CountMetricE2eTest, TestInitialConditionChanges);
648
649 FRIEND_TEST(DurationMetricE2eTest, TestOneBucket);
650 FRIEND_TEST(DurationMetricE2eTest, TestTwoBuckets);
651 FRIEND_TEST(DurationMetricE2eTest, TestWithActivation);
652 FRIEND_TEST(DurationMetricE2eTest, TestWithCondition);
653 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition);
654 FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition);
655 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedState);
656 FRIEND_TEST(DurationMetricE2eTest, TestWithConditionAndSlicedState);
657 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStateMapped);
658 FRIEND_TEST(DurationMetricE2eTest, TestSlicedStatePrimaryFieldsNotSubsetDimInWhat);
659 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStatePrimaryFieldsSubset);
660 FRIEND_TEST(DurationMetricE2eTest, TestUploadThreshold);
661
662 FRIEND_TEST(MetricActivationE2eTest, TestCountMetric);
663 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithOneDeactivation);
664 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoDeactivations);
665 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithSameDeactivation);
666 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoMetricsTwoDeactivations);
667
668 FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead);
669 FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBoot);
670 FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBootMultipleActivations);
671 FRIEND_TEST(StatsLogProcessorTest,
672 TestActivationOnBootMultipleActivationsDifferentActivationTypes);
673 FRIEND_TEST(StatsLogProcessorTest, TestActivationsPersistAcrossSystemServerRestart);
674
675 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState);
676 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions);
677 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions);
678 FRIEND_TEST(ValueMetricE2eTest, TestInitialConditionChanges);
679
680 FRIEND_TEST(MetricsManagerUtilTest, TestInitialConditions);
681 FRIEND_TEST(MetricsManagerUtilTest, TestSampledMetrics);
682 FRIEND_TEST(MetricsManagerUtilTest, TestUidFields);
683
684 FRIEND_TEST(ConfigUpdateTest, TestUpdateMetricActivations);
685 FRIEND_TEST(ConfigUpdateTest, TestUpdateCountMetrics);
686 FRIEND_TEST(ConfigUpdateTest, TestUpdateEventMetrics);
687 FRIEND_TEST(ConfigUpdateTest, TestUpdateGaugeMetrics);
688 FRIEND_TEST(ConfigUpdateTest, TestUpdateDurationMetrics);
689 FRIEND_TEST(ConfigUpdateTest, TestUpdateMetricsMultipleTypes);
690 FRIEND_TEST(ConfigUpdateTest, TestUpdateAlerts);
691
692 FRIEND_TEST(EventMetricProducerTest, TestCorruptedDataReason_OnDumpReport);
693 FRIEND_TEST(EventMetricProducerTest, TestCorruptedDataReason_OnDropData);
694 FRIEND_TEST(EventMetricProducerTest, TestCorruptedDataReason_OnClearPastBuckets);
695 FRIEND_TEST(EventMetricProducerTest, TestCorruptedDataReason_UnrecoverableLossOfCondition);
696 };
697
698 } // namespace statsd
699 } // namespace os
700 } // namespace android
701 #endif // METRIC_PRODUCER_H
702