1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef METRIC_PRODUCER_H
18 #define METRIC_PRODUCER_H
19 
20 #include <src/active_config_list.pb.h>
21 #include <src/guardrail/stats_log_enums.pb.h>
22 #include <utils/RefBase.h>
23 
24 #include <unordered_map>
25 
26 #include "HashableDimensionKey.h"
27 #include "anomaly/AnomalyTracker.h"
28 #include "condition/ConditionTimer.h"
29 #include "condition/ConditionWizard.h"
30 #include "config/ConfigKey.h"
31 #include "config/ConfigMetadataProvider.h"
32 #include "guardrail/StatsdStats.h"
33 #include "matchers/EventMatcherWizard.h"
34 #include "matchers/matcher_util.h"
35 #include "packages/PackageInfoListener.h"
36 #include "src/statsd_metadata.pb.h"  // MetricMetadata
37 #include "state/StateListener.h"
38 #include "state/StateManager.h"
39 #include "utils/DbUtils.h"
40 #include "utils/ShardOffsetProvider.h"
41 #include "utils/api_tracing.h"
42 
43 namespace android {
44 namespace os {
45 namespace statsd {
46 
47 // If the metric has no activation requirement, it will be active once the metric producer is
48 // created.
49 // If the metric needs to be activated by atoms, the metric producer will start
50 // with kNotActive state, turn to kActive or kActiveOnBoot when the activation event arrives, become
51 // kNotActive when it reaches the duration limit (timebomb). If the activation event arrives again
52 // before or after it expires, the event producer will be re-activated and ttl will be reset.
53 enum ActivationState {
54     kNotActive = 0,
55     kActive = 1,
56     kActiveOnBoot = 2,
57 };
58 
59 enum DumpLatency {
60     // In some cases, we only have a short time range to do the dump, e.g. statsd is being killed.
61     // We might be able to return all the data in this mode. For instance, pull metrics might need
62     // to be pulled when the current bucket is requested.
63     FAST = 1,
64     // In other cases, it is fine for a dump to take more than a few milliseconds, e.g. config
65     // updates.
66     NO_TIME_CONSTRAINTS = 2
67 };
68 
69 enum MetricType {
70     METRIC_TYPE_EVENT = 1,
71     METRIC_TYPE_COUNT = 2,
72     METRIC_TYPE_DURATION = 3,
73     METRIC_TYPE_GAUGE = 4,
74     METRIC_TYPE_VALUE = 5,
75     METRIC_TYPE_KLL = 6,
76 };
77 
78 struct Activation {
ActivationActivation79     Activation(const ActivationType& activationType, int64_t ttlNs)
80         : ttl_ns(ttlNs),
81           start_ns(0),
82           state(ActivationState::kNotActive),
83           activationType(activationType) {
84     }
85 
86     const int64_t ttl_ns;
87     int64_t start_ns;
88     ActivationState state;
89     const ActivationType activationType;
90 };
91 
92 struct DropEvent {
93     // Reason for dropping the bucket and/or marking the bucket invalid.
94     BucketDropReason reason;
95     // The timestamp of the drop event.
96     int64_t dropTimeNs;
97 };
98 
99 struct SkippedBucket {
100     // Start time of the dropped bucket.
101     int64_t bucketStartTimeNs;
102     // End time of the dropped bucket.
103     int64_t bucketEndTimeNs;
104     // List of events that invalidated this bucket.
105     std::vector<DropEvent> dropEvents;
106 
resetSkippedBucket107     void reset() {
108         bucketStartTimeNs = 0;
109         bucketEndTimeNs = 0;
110         dropEvents.clear();
111     }
112 };
113 
114 struct SamplingInfo {
115     // Matchers for sampled fields. Currently only one sampled dimension is supported.
116     std::vector<Matcher> sampledWhatFields;
117 
118     int shardCount = 0;
119 };
120 
121 template <class T>
getAppUpgradeBucketSplit(const T & metric)122 optional<bool> getAppUpgradeBucketSplit(const T& metric) {
123     return metric.has_split_bucket_for_app_upgrade()
124                    ? std::make_optional<bool>(metric.split_bucket_for_app_upgrade())
125                    : std::nullopt;
126 }
127 
128 // A MetricProducer is responsible for compute one single metric, creating stats log report, and
129 // writing the report to dropbox. MetricProducers should respond to package changes as required in
130 // PackageInfoListener, but if none of the metrics are slicing by package name, then the update can
131 // be a no-op.
132 class MetricProducer : public virtual RefBase, public virtual StateListener {
133 public:
134     MetricProducer(int64_t metricId, const ConfigKey& key, int64_t timeBaseNs,
135                    const int conditionIndex, const vector<ConditionState>& initialConditionCache,
136                    const sp<ConditionWizard>& wizard, const uint64_t protoHash,
137                    const std::unordered_map<int, std::shared_ptr<Activation>>& eventActivationMap,
138                    const std::unordered_map<int, std::vector<std::shared_ptr<Activation>>>&
139                            eventDeactivationMap,
140                    const vector<int>& slicedStateAtoms,
141                    const unordered_map<int, unordered_map<int, int64_t>>& stateGroupMap,
142                    const optional<bool> splitBucketForAppUpgrade,
143                    const wp<ConfigMetadataProvider> configMetadataProvider);
144 
~MetricProducer()145     virtual ~MetricProducer(){};
146 
initialCondition(const int conditionIndex,const vector<ConditionState> & initialConditionCache)147     ConditionState initialCondition(const int conditionIndex,
148                                     const vector<ConditionState>& initialConditionCache) const {
149         return conditionIndex >= 0 ? initialConditionCache[conditionIndex] : ConditionState::kTrue;
150     }
151 
152     // Update appropriate state on config updates. Primarily, all indices need to be updated.
153     // This metric and all of its dependencies are guaranteed to be preserved across the update.
154     // This function also updates several maps used by metricsManager.
155     // This function clears all anomaly trackers. All anomaly trackers need to be added again.
onConfigUpdated(const StatsdConfig & config,int configIndex,int metricIndex,const std::vector<sp<AtomMatchingTracker>> & allAtomMatchingTrackers,const std::unordered_map<int64_t,int> & oldAtomMatchingTrackerMap,const std::unordered_map<int64_t,int> & newAtomMatchingTrackerMap,const sp<EventMatcherWizard> & matcherWizard,const std::vector<sp<ConditionTracker>> & allConditionTrackers,const std::unordered_map<int64_t,int> & conditionTrackerMap,const sp<ConditionWizard> & wizard,const std::unordered_map<int64_t,int> & metricToActivationMap,std::unordered_map<int,std::vector<int>> & trackerToMetricMap,std::unordered_map<int,std::vector<int>> & conditionToMetricMap,std::unordered_map<int,std::vector<int>> & activationAtomTrackerToMetricMap,std::unordered_map<int,std::vector<int>> & deactivationAtomTrackerToMetricMap,std::vector<int> & metricsWithActivation)156     optional<InvalidConfigReason> onConfigUpdated(
157             const StatsdConfig& config, int configIndex, int metricIndex,
158             const std::vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers,
159             const std::unordered_map<int64_t, int>& oldAtomMatchingTrackerMap,
160             const std::unordered_map<int64_t, int>& newAtomMatchingTrackerMap,
161             const sp<EventMatcherWizard>& matcherWizard,
162             const std::vector<sp<ConditionTracker>>& allConditionTrackers,
163             const std::unordered_map<int64_t, int>& conditionTrackerMap,
164             const sp<ConditionWizard>& wizard,
165             const std::unordered_map<int64_t, int>& metricToActivationMap,
166             std::unordered_map<int, std::vector<int>>& trackerToMetricMap,
167             std::unordered_map<int, std::vector<int>>& conditionToMetricMap,
168             std::unordered_map<int, std::vector<int>>& activationAtomTrackerToMetricMap,
169             std::unordered_map<int, std::vector<int>>& deactivationAtomTrackerToMetricMap,
170             std::vector<int>& metricsWithActivation) {
171         std::lock_guard<std::mutex> lock(mMutex);
172         return onConfigUpdatedLocked(config, configIndex, metricIndex, allAtomMatchingTrackers,
173                                      oldAtomMatchingTrackerMap, newAtomMatchingTrackerMap,
174                                      matcherWizard, allConditionTrackers, conditionTrackerMap,
175                                      wizard, metricToActivationMap, trackerToMetricMap,
176                                      conditionToMetricMap, activationAtomTrackerToMetricMap,
177                                      deactivationAtomTrackerToMetricMap, metricsWithActivation);
178     };
179 
180     /**
181      * Force a partial bucket split on app upgrade
182      */
notifyAppUpgrade(int64_t eventTimeNs)183     void notifyAppUpgrade(int64_t eventTimeNs) {
184         std::lock_guard<std::mutex> lock(mMutex);
185         const bool splitBucket =
186                 mSplitBucketForAppUpgrade ? mSplitBucketForAppUpgrade.value() : false;
187         if (!splitBucket) {
188             return;
189         }
190         notifyAppUpgradeInternalLocked(eventTimeNs);
191     };
192 
notifyAppRemoved(int64_t eventTimeNs)193     void notifyAppRemoved(int64_t eventTimeNs) {
194         // Force buckets to split on removal also.
195         notifyAppUpgrade(eventTimeNs);
196     };
197 
198     /**
199      * Force a partial bucket split on boot complete.
200      */
onStatsdInitCompleted(int64_t eventTimeNs)201     virtual void onStatsdInitCompleted(int64_t eventTimeNs) {
202         ATRACE_CALL();
203         std::lock_guard<std::mutex> lock(mMutex);
204         flushLocked(eventTimeNs);
205     }
206 
207     // Consume the parsed stats log entry that already matched the "what" of the metric.
onMatchedLogEvent(const size_t matcherIndex,const LogEvent & event)208     void onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) {
209         std::lock_guard<std::mutex> lock(mMutex);
210         onMatchedLogEventLocked(matcherIndex, event);
211     }
212 
213     enum class LostAtomType {
214         kWhat = 0,
215         kCondition,
216         kState,
217     };
218 
onMatchedLogEventLost(int32_t atomId,DataCorruptedReason reason,LostAtomType atomType)219     void onMatchedLogEventLost(int32_t atomId, DataCorruptedReason reason, LostAtomType atomType) {
220         std::lock_guard<std::mutex> lock(mMutex);
221         onMatchedLogEventLostLocked(atomId, reason, atomType);
222     }
223 
onConditionChanged(const bool condition,int64_t eventTime)224     void onConditionChanged(const bool condition, int64_t eventTime) {
225         std::lock_guard<std::mutex> lock(mMutex);
226         onConditionChangedLocked(condition, eventTime);
227     }
228 
onSlicedConditionMayChange(bool overallCondition,int64_t eventTime)229     void onSlicedConditionMayChange(bool overallCondition, int64_t eventTime) {
230         std::lock_guard<std::mutex> lock(mMutex);
231         onSlicedConditionMayChangeLocked(overallCondition, eventTime);
232     }
233 
isConditionSliced()234     bool isConditionSliced() const {
235         std::lock_guard<std::mutex> lock(mMutex);
236         return mConditionSliced;
237     };
238 
onStateChanged(const int64_t eventTimeNs,const int32_t atomId,const HashableDimensionKey & primaryKey,const FieldValue & oldState,const FieldValue & newState)239     void onStateChanged(const int64_t eventTimeNs, const int32_t atomId,
240                         const HashableDimensionKey& primaryKey, const FieldValue& oldState,
241                         const FieldValue& newState){};
242 
onStateEventLost(int32_t atomId,DataCorruptedReason reason)243     void onStateEventLost(int32_t atomId, DataCorruptedReason reason) override {
244         std::lock_guard<std::mutex> lock(mMutex);
245         onMatchedLogEventLostLocked(atomId, reason, LostAtomType::kState);
246     }
247 
248     // Output the metrics data to [protoOutput]. All metrics reports end with the same timestamp.
249     // This method clears all the past buckets.
onDumpReport(const int64_t dumpTimeNs,const bool include_current_partial_bucket,const bool erase_data,const DumpLatency dumpLatency,std::set<string> * str_set,std::set<int32_t> & usedUids,android::util::ProtoOutputStream * protoOutput)250     void onDumpReport(const int64_t dumpTimeNs, const bool include_current_partial_bucket,
251                       const bool erase_data, const DumpLatency dumpLatency,
252                       std::set<string>* str_set, std::set<int32_t>& usedUids,
253                       android::util::ProtoOutputStream* protoOutput) {
254         std::lock_guard<std::mutex> lock(mMutex);
255         onDumpReportLocked(dumpTimeNs, include_current_partial_bucket, erase_data, dumpLatency,
256                            str_set, usedUids, protoOutput);
257     }
258 
259     virtual optional<InvalidConfigReason> onConfigUpdatedLocked(
260             const StatsdConfig& config, int configIndex, int metricIndex,
261             const std::vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers,
262             const std::unordered_map<int64_t, int>& oldAtomMatchingTrackerMap,
263             const std::unordered_map<int64_t, int>& newAtomMatchingTrackerMap,
264             const sp<EventMatcherWizard>& matcherWizard,
265             const std::vector<sp<ConditionTracker>>& allConditionTrackers,
266             const std::unordered_map<int64_t, int>& conditionTrackerMap,
267             const sp<ConditionWizard>& wizard,
268             const std::unordered_map<int64_t, int>& metricToActivationMap,
269             std::unordered_map<int, std::vector<int>>& trackerToMetricMap,
270             std::unordered_map<int, std::vector<int>>& conditionToMetricMap,
271             std::unordered_map<int, std::vector<int>>& activationAtomTrackerToMetricMap,
272             std::unordered_map<int, std::vector<int>>& deactivationAtomTrackerToMetricMap,
273             std::vector<int>& metricsWithActivation);
274 
clearPastBuckets(const int64_t dumpTimeNs)275     void clearPastBuckets(const int64_t dumpTimeNs) {
276         std::lock_guard<std::mutex> lock(mMutex);
277         clearPastBucketsLocked(dumpTimeNs);
278     }
279 
prepareFirstBucket()280     void prepareFirstBucket() {
281         std::lock_guard<std::mutex> lock(mMutex);
282         prepareFirstBucketLocked();
283     }
284 
285     // Returns the memory in bytes currently used to store this metric's data. Does not change
286     // state.
byteSize()287     size_t byteSize() const {
288         std::lock_guard<std::mutex> lock(mMutex);
289         return byteSizeLocked();
290     }
291 
dumpStates(int out,bool verbose)292     void dumpStates(int out, bool verbose) const {
293         std::lock_guard<std::mutex> lock(mMutex);
294         dumpStatesLocked(out, verbose);
295     }
296 
297     // Let MetricProducer drop in-memory data to save memory.
298     // We still need to keep future data valid and anomaly tracking work, which means we will
299     // have to flush old data, informing anomaly trackers then safely drop old data.
300     // We still keep current bucket data for future metrics' validity.
dropData(const int64_t dropTimeNs)301     void dropData(const int64_t dropTimeNs) {
302         std::lock_guard<std::mutex> lock(mMutex);
303         dropDataLocked(dropTimeNs);
304     }
305 
loadActiveMetric(const ActiveMetric & activeMetric,int64_t currentTimeNs)306     void loadActiveMetric(const ActiveMetric& activeMetric, int64_t currentTimeNs) {
307         std::lock_guard<std::mutex> lock(mMutex);
308         loadActiveMetricLocked(activeMetric, currentTimeNs);
309     }
310 
activate(int activationTrackerIndex,int64_t elapsedTimestampNs)311     void activate(int activationTrackerIndex, int64_t elapsedTimestampNs) {
312         std::lock_guard<std::mutex> lock(mMutex);
313         activateLocked(activationTrackerIndex, elapsedTimestampNs);
314     }
315 
cancelEventActivation(int deactivationTrackerIndex)316     void cancelEventActivation(int deactivationTrackerIndex) {
317         std::lock_guard<std::mutex> lock(mMutex);
318         cancelEventActivationLocked(deactivationTrackerIndex);
319     }
320 
isActive()321     bool isActive() const {
322         std::lock_guard<std::mutex> lock(mMutex);
323         return isActiveLocked();
324     }
325 
326     void flushIfExpire(int64_t elapsedTimestampNs);
327 
328     void writeActiveMetricToProtoOutputStream(int64_t currentTimeNs, const DumpReportReason reason,
329                                               ProtoOutputStream* proto);
330 
enforceRestrictedDataTtl(sqlite3 * db,int64_t wallClockNs)331     virtual void enforceRestrictedDataTtl(sqlite3* db, int64_t wallClockNs){};
332 
writeMetricMetadataToProto(metadata::MetricMetadata * metricMetadata)333     virtual bool writeMetricMetadataToProto(metadata::MetricMetadata* metricMetadata) {
334         return false;
335     }
336 
loadMetricMetadataFromProto(const metadata::MetricMetadata & metricMetadata)337     virtual void loadMetricMetadataFromProto(const metadata::MetricMetadata& metricMetadata){};
338 
339     /* Called when the metric is to about to be removed from config. */
onMetricRemove()340     virtual void onMetricRemove() {
341     }
342 
flushRestrictedData()343     virtual void flushRestrictedData() {
344     }
345 
346     // Start: getters/setters
getMetricId()347     inline int64_t getMetricId() const {
348         return mMetricId;
349     }
350 
getProtoHash()351     inline uint64_t getProtoHash() const {
352         return mProtoHash;
353     }
354 
355     virtual MetricType getMetricType() const = 0;
356 
357     // For test only.
getCurrentBucketNum()358     inline int64_t getCurrentBucketNum() const {
359         return mCurrentBucketNum;
360     }
361 
getSlicedStateAtoms()362     inline const std::vector<int> getSlicedStateAtoms() {
363         std::lock_guard<std::mutex> lock(mMutex);
364         return mSlicedStateAtoms;
365     }
366 
isValid()367     inline bool isValid() const {
368         return mValid;
369     }
370 
371     /* Adds an AnomalyTracker and returns it. */
addAnomalyTracker(const Alert & alert,const sp<AlarmMonitor> & anomalyAlarmMonitor,const UpdateStatus & updateStatus,const int64_t updateTimeNs)372     virtual sp<AnomalyTracker> addAnomalyTracker(const Alert& alert,
373                                                  const sp<AlarmMonitor>& anomalyAlarmMonitor,
374                                                  const UpdateStatus& updateStatus,
375                                                  const int64_t updateTimeNs) {
376         std::lock_guard<std::mutex> lock(mMutex);
377         sp<AnomalyTracker> anomalyTracker = new AnomalyTracker(alert, mConfigKey);
378         mAnomalyTrackers.push_back(anomalyTracker);
379         return anomalyTracker;
380     }
381 
382     /* Adds an AnomalyTracker that has already been created */
addAnomalyTracker(sp<AnomalyTracker> & anomalyTracker,int64_t updateTimeNs)383     virtual void addAnomalyTracker(sp<AnomalyTracker>& anomalyTracker, int64_t updateTimeNs) {
384         std::lock_guard<std::mutex> lock(mMutex);
385         mAnomalyTrackers.push_back(anomalyTracker);
386     }
387 
setSamplingInfo(SamplingInfo samplingInfo)388     void setSamplingInfo(SamplingInfo samplingInfo) {
389         std::lock_guard<std::mutex> lock(mMutex);
390         mSampledWhatFields.swap(samplingInfo.sampledWhatFields);
391         mShardCount = samplingInfo.shardCount;
392     }
393 
setUidFields(std::vector<Matcher> uidFields)394     void setUidFields(std::vector<Matcher> uidFields) {
395         std::lock_guard<std::mutex> lock(mMutex);
396         mUidFields.swap(uidFields);
397     }
398 
399     // End: getters/setters
400 protected:
401     /**
402      * Flushes the current bucket if the eventTime is after the current bucket's end time.
403      */
flushIfNeededLocked(int64_t eventTime)404     virtual void flushIfNeededLocked(int64_t eventTime){};
405 
406     /**
407      * For metrics that aggregate (ie, every metric producer except for EventMetricProducer),
408      * we need to be able to flush the current buckets on demand (ie, end the current bucket and
409      * start new bucket). If this function is called when eventTimeNs is greater than the current
410      * bucket's end timestamp, than we flush up to the end of the latest full bucket; otherwise,
411      * we assume that we want to flush a partial bucket. The bucket start timestamp and bucket
412      * number are not changed by this function. This method should only be called by
413      * flushIfNeededLocked or flushLocked or the app upgrade handler; the caller MUST update the
414      * bucket timestamp and bucket number as needed.
415      */
flushCurrentBucketLocked(int64_t eventTimeNs,int64_t nextBucketStartTimeNs)416     virtual void flushCurrentBucketLocked(int64_t eventTimeNs, int64_t nextBucketStartTimeNs){};
417 
418     /**
419      * Flushes all the data including the current partial bucket.
420      */
flushLocked(int64_t eventTimeNs)421     void flushLocked(int64_t eventTimeNs) {
422         flushIfNeededLocked(eventTimeNs);
423         flushCurrentBucketLocked(eventTimeNs, eventTimeNs);
424     };
425 
notifyAppUpgradeInternalLocked(const int64_t eventTimeNs)426     virtual void notifyAppUpgradeInternalLocked(const int64_t eventTimeNs) {
427         flushLocked(eventTimeNs);
428     }
429 
430     /*
431      * Individual metrics can implement their own business logic here. All pre-processing is done.
432      *
433      * [matcherIndex]: the index of the matcher which matched this event. This is interesting to
434      *                 DurationMetric, because it has start/stop/stop_all 3 matchers.
435      * [eventKey]: the extracted dimension key for the final output. if the metric doesn't have
436      *             dimensions, it will be DEFAULT_DIMENSION_KEY
437      * [conditionKey]: the keys of conditions which should be used to query the condition for this
438      *                 target event (from MetricConditionLink). This is passed to individual metrics
439      *                 because DurationMetric needs it to be cached.
440      * [condition]: whether condition is met. If condition is sliced, this is the result coming from
441      *              query with ConditionWizard; If condition is not sliced, this is the
442      *              nonSlicedCondition.
443      * [event]: the log event, just in case the metric needs its data, e.g., EventMetric.
444      */
445     virtual void onMatchedLogEventInternalLocked(
446             const size_t matcherIndex, const MetricDimensionKey& eventKey,
447             const ConditionKey& conditionKey, bool condition, const LogEvent& event,
448             const map<int, HashableDimensionKey>& statePrimaryKeys) = 0;
449 
450     // Consume the parsed stats log entry that already matched the "what" of the metric.
451     virtual void onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event);
452     virtual void onMatchedLogEventLostLocked(int32_t atomId, DataCorruptedReason reason,
453                                              LostAtomType atomType);
454     virtual void onConditionChangedLocked(const bool condition, int64_t eventTime) = 0;
455     virtual void onSlicedConditionMayChangeLocked(bool overallCondition,
456                                                   const int64_t eventTime) = 0;
457     virtual void onDumpReportLocked(const int64_t dumpTimeNs,
458                                     const bool include_current_partial_bucket,
459                                     const bool erase_data, const DumpLatency dumpLatency,
460                                     std::set<string>* str_set, std::set<int32_t>& usedUids,
461                                     android::util::ProtoOutputStream* protoOutput) = 0;
462     virtual void clearPastBucketsLocked(const int64_t dumpTimeNs) = 0;
prepareFirstBucketLocked()463     virtual void prepareFirstBucketLocked(){};
464     virtual size_t byteSizeLocked() const = 0;
465     virtual void dumpStatesLocked(int out, bool verbose) const = 0;
466     virtual void dropDataLocked(const int64_t dropTimeNs) = 0;
467     void loadActiveMetricLocked(const ActiveMetric& activeMetric, int64_t currentTimeNs);
468     void activateLocked(int activationTrackerIndex, int64_t elapsedTimestampNs);
469     void cancelEventActivationLocked(int deactivationTrackerIndex);
470 
471     // Computes the size of a newly added bucket to this metric, taking into account any new
472     // dimensions that are introduced if necessary.
473     virtual size_t computeBucketSizeLocked(const bool isFullBucket,
474                                            const MetricDimensionKey& dimKey,
475                                            const bool isFirstBucket) const;
476     size_t computeOverheadSizeLocked(const bool hasPastBuckets,
477                                      const bool dimensionGuardrailHit) const;
478     size_t computeSkippedBucketSizeLocked(const SkippedBucket& skippedBucket) const;
479 
480     bool evaluateActiveStateLocked(int64_t elapsedTimestampNs);
481 
onActiveStateChangedLocked(const int64_t eventTimeNs,const bool isActive)482     virtual void onActiveStateChangedLocked(const int64_t eventTimeNs, const bool isActive) {
483         if (!isActive) {
484             flushLocked(eventTimeNs);
485         }
486     }
487 
isActiveLocked()488     inline bool isActiveLocked() const {
489         return mIsActive;
490     }
491 
492     // Convenience to compute the current bucket's end time, which is always aligned with the
493     // start time of the metric.
getCurrentBucketEndTimeNs()494     int64_t getCurrentBucketEndTimeNs() const {
495         return mTimeBaseNs + (mCurrentBucketNum + 1) * mBucketSizeNs;
496     }
497 
getBucketNumFromEndTimeNs(const int64_t endNs)498     int64_t getBucketNumFromEndTimeNs(const int64_t endNs) {
499         return (endNs - mTimeBaseNs) / mBucketSizeNs - 1;
500     }
501 
502     // Query StateManager for original state value using the queryKey.
503     // The field and value are output.
504     void queryStateValue(int32_t atomId, const HashableDimensionKey& queryKey, FieldValue* value);
505 
506     // If a state map exists for the given atom, replace the original state
507     // value with the group id mapped to the value.
508     // If no state map exists, keep the original state value.
509     void mapStateValue(int32_t atomId, FieldValue* value);
510 
511     // Returns a HashableDimensionKey with unknown state value for each state
512     // atom.
513     HashableDimensionKey getUnknownStateKey();
514 
515     DropEvent buildDropEvent(const int64_t dropTimeNs, const BucketDropReason reason) const;
516 
517     // Returns true if the number of drop events in the current bucket has
518     // exceeded the maximum number allowed, which is currently capped at 10.
519     bool maxDropEventsReached() const;
520 
521     bool passesSampleCheckLocked(const vector<FieldValue>& values) const;
522 
523     const int64_t mMetricId;
524 
525     // Hash of the Metric's proto bytes from StatsdConfig, including any activations.
526     // Used to determine if the definition of this metric has changed across a config update.
527     const uint64_t mProtoHash;
528 
529     const ConfigKey mConfigKey;
530 
531     bool mValid;
532 
533     // The time when this metric producer was first created. The end time for the current bucket
534     // can be computed from this based on mCurrentBucketNum.
535     const int64_t mTimeBaseNs;
536 
537     // Start time may not be aligned with the start of statsd if there is an app upgrade in the
538     // middle of a bucket.
539     int64_t mCurrentBucketStartTimeNs;
540 
541     // Used by anomaly detector to track which bucket we are in. This is not sent with the produced
542     // report.
543     int64_t mCurrentBucketNum;
544 
545     int64_t mBucketSizeNs;
546 
547     ConditionState mCondition;
548 
549     ConditionTimer mConditionTimer;
550 
551     int mConditionTrackerIndex;
552 
553     // TODO(b/185770739): use !mMetric2ConditionLinks.empty()
554     bool mConditionSliced;
555 
556     sp<ConditionWizard> mWizard;
557 
558     bool mContainANYPositionInDimensionsInWhat;
559 
560     // Metrics slicing by primitive repeated field and/or position ALL need to use nested
561     // dimensions.
562     bool mShouldUseNestedDimensions;
563 
564     vector<Matcher> mDimensionsInWhat;  // The dimensions_in_what defined in statsd_config
565 
566     // True iff the metric to condition links cover all dimension fields in the condition tracker.
567     // This field is always false for combinational condition trackers.
568     bool mHasLinksToAllConditionDimensionsInTracker;
569 
570     std::vector<Metric2Condition> mMetric2ConditionLinks;
571 
572     std::vector<sp<AnomalyTracker>> mAnomalyTrackers;
573 
574     mutable std::mutex mMutex;
575 
576     // When the metric producer has multiple activations, these activations are ORed to determine
577     // whether the metric producer is ready to generate metrics.
578     std::unordered_map<int, std::shared_ptr<Activation>> mEventActivationMap;
579 
580     // Maps index of atom matcher for deactivation to a list of Activation structs.
581     std::unordered_map<int, std::vector<std::shared_ptr<Activation>>> mEventDeactivationMap;
582 
583     bool mIsActive;
584 
585     // The slice_by_state atom ids defined in statsd_config.
586     const std::vector<int32_t> mSlicedStateAtoms;
587 
588     // Maps atom ids and state values to group_ids (<atom_id, <value, group_id>>).
589     const std::unordered_map<int32_t, std::unordered_map<int, int64_t>> mStateGroupMap;
590 
591     // MetricStateLinks defined in statsd_config that link fields in the state
592     // atom to fields in the "what" atom.
593     std::vector<Metric2State> mMetric2StateLinks;
594 
595     optional<UploadThreshold> mUploadThreshold;
596 
597     const optional<bool> mSplitBucketForAppUpgrade;
598 
599     SkippedBucket mCurrentSkippedBucket;
600     // Buckets that were invalidated and had their data dropped.
601     std::vector<SkippedBucket> mSkippedBuckets;
602 
603     // If hard dimension guardrail is hit, do not spam logcat. This is a per bucket tracker.
604     mutable bool mHasHitGuardrail;
605 
606     // Matchers for sampled fields. Currently only one sampled dimension is supported.
607     std::vector<Matcher> mSampledWhatFields;
608 
609     int mShardCount;
610 
611     // For tracking uid fields in a metric. Only needed if the field is not annotated in the atom
612     // and omit_unused_uids_in_uidmap = true.
613     std::vector<Matcher> mUidFields;
614 
615     sp<ConfigMetadataProvider> getConfigMetadataProvider() const;
616 
617     wp<ConfigMetadataProvider> mConfigMetadataProvider;
618 
619     enum DataCorruptionSeverity { kNone = 0, kResetOnDump, kUnrecoverable };
620 
621     DataCorruptionSeverity mDataCorruptedDueToSocketLoss = DataCorruptionSeverity::kNone;
622     DataCorruptionSeverity mDataCorruptedDueToQueueOverflow = DataCorruptionSeverity::kNone;
623 
624     /**
625      * @brief Determines DataCorruptionSeverity based on source and atom type being lost
626      *
627      * @return DataCorruptionSeverity
628      */
determineCorruptionSeverity(int32_t,DataCorruptedReason,LostAtomType)629     virtual DataCorruptionSeverity determineCorruptionSeverity(int32_t /*atomId*/,
630                                                                DataCorruptedReason /*reason*/,
631                                                                LostAtomType /*atomType*/) const {
632         return DataCorruptionSeverity::kNone;
633     };
634 
635     /**
636      * @brief Resets data corruption reason info if no unrecoverable errors observed
637      */
638     void resetDataCorruptionFlagsLocked();
639 
640     size_t mTotalDataSize = 0;
641 
642     friend class SocketLossInfoTest;
643     FRIEND_TEST(CountMetricE2eTest, TestSlicedState);
644     FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithMap);
645     FRIEND_TEST(CountMetricE2eTest, TestMultipleSlicedStates);
646     FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithPrimaryFields);
647     FRIEND_TEST(CountMetricE2eTest, TestInitialConditionChanges);
648 
649     FRIEND_TEST(DurationMetricE2eTest, TestOneBucket);
650     FRIEND_TEST(DurationMetricE2eTest, TestTwoBuckets);
651     FRIEND_TEST(DurationMetricE2eTest, TestWithActivation);
652     FRIEND_TEST(DurationMetricE2eTest, TestWithCondition);
653     FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition);
654     FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition);
655     FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedState);
656     FRIEND_TEST(DurationMetricE2eTest, TestWithConditionAndSlicedState);
657     FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStateMapped);
658     FRIEND_TEST(DurationMetricE2eTest, TestSlicedStatePrimaryFieldsNotSubsetDimInWhat);
659     FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStatePrimaryFieldsSubset);
660     FRIEND_TEST(DurationMetricE2eTest, TestUploadThreshold);
661 
662     FRIEND_TEST(MetricActivationE2eTest, TestCountMetric);
663     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithOneDeactivation);
664     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoDeactivations);
665     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithSameDeactivation);
666     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoMetricsTwoDeactivations);
667 
668     FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead);
669     FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBoot);
670     FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBootMultipleActivations);
671     FRIEND_TEST(StatsLogProcessorTest,
672                 TestActivationOnBootMultipleActivationsDifferentActivationTypes);
673     FRIEND_TEST(StatsLogProcessorTest, TestActivationsPersistAcrossSystemServerRestart);
674 
675     FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState);
676     FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions);
677     FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions);
678     FRIEND_TEST(ValueMetricE2eTest, TestInitialConditionChanges);
679 
680     FRIEND_TEST(MetricsManagerUtilTest, TestInitialConditions);
681     FRIEND_TEST(MetricsManagerUtilTest, TestSampledMetrics);
682     FRIEND_TEST(MetricsManagerUtilTest, TestUidFields);
683 
684     FRIEND_TEST(ConfigUpdateTest, TestUpdateMetricActivations);
685     FRIEND_TEST(ConfigUpdateTest, TestUpdateCountMetrics);
686     FRIEND_TEST(ConfigUpdateTest, TestUpdateEventMetrics);
687     FRIEND_TEST(ConfigUpdateTest, TestUpdateGaugeMetrics);
688     FRIEND_TEST(ConfigUpdateTest, TestUpdateDurationMetrics);
689     FRIEND_TEST(ConfigUpdateTest, TestUpdateMetricsMultipleTypes);
690     FRIEND_TEST(ConfigUpdateTest, TestUpdateAlerts);
691 
692     FRIEND_TEST(EventMetricProducerTest, TestCorruptedDataReason_OnDumpReport);
693     FRIEND_TEST(EventMetricProducerTest, TestCorruptedDataReason_OnDropData);
694     FRIEND_TEST(EventMetricProducerTest, TestCorruptedDataReason_OnClearPastBuckets);
695     FRIEND_TEST(EventMetricProducerTest, TestCorruptedDataReason_UnrecoverableLossOfCondition);
696 };
697 
698 }  // namespace statsd
699 }  // namespace os
700 }  // namespace android
701 #endif  // METRIC_PRODUCER_H
702