xref: /aosp_15_r20/frameworks/av/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp (revision ec779b8e0859a360c3d303172224686826e6e0e1) 
1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "Camera3-OutputUtils"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20 //#define LOG_NNDEBUG 0  // Per-frame verbose logging
21 
22 #ifdef LOG_NNDEBUG
23 #define ALOGVV(...) ALOGV(__VA_ARGS__)
24 #else
25 #define ALOGVV(...) ((void)0)
26 #endif
27 
28 // Convenience macros for transitioning to the error state
29 #define SET_ERR(fmt, ...) states.setErrIntf.setErrorState(   \
30     "%s: " fmt, __FUNCTION__,                         \
31     ##__VA_ARGS__)
32 
33 #include <inttypes.h>
34 
35 #include <utils/Log.h>
36 #include <utils/SortedVector.h>
37 #include <utils/Trace.h>
38 
39 #include <android/hardware/ICameraService.h>
40 #include <android/hardware/camera2/ICameraDeviceCallbacks.h>
41 
42 #include <android/hardware/camera/device/3.4/ICameraDeviceCallback.h>
43 #include <android/hardware/camera/device/3.5/ICameraDeviceCallback.h>
44 #include <android/hardware/camera/device/3.5/ICameraDeviceSession.h>
45 
46 #include <camera/CameraUtils.h>
47 #include <camera/StringUtils.h>
48 #include <camera_metadata_hidden.h>
49 #include <com_android_internal_camera_flags.h>
50 
51 #include "device3/Camera3OutputUtils.h"
52 #include "utils/SessionConfigurationUtils.h"
53 
54 #include "system/camera_metadata.h"
55 
56 using namespace android::camera3;
57 using namespace android::camera3::SessionConfigurationUtils;
58 using namespace android::hardware::camera;
59 using CameraMetadataInfo = android::hardware::camera2::CameraMetadataInfo;
60 namespace flags = com::android::internal::camera::flags;
61 
62 namespace android {
63 namespace camera3 {
64 
fixupMonochromeTags(CaptureOutputStates & states,const CameraMetadata & deviceInfo,CameraMetadata & resultMetadata)65 status_t fixupMonochromeTags(
66         CaptureOutputStates& states,
67         const CameraMetadata& deviceInfo,
68         CameraMetadata& resultMetadata) {
69     status_t res = OK;
70     if (!states.needFixupMonoChrome) {
71         return res;
72     }
73 
74     // Remove tags that are not applicable to monochrome camera.
75     int32_t tagsToRemove[] = {
76            ANDROID_SENSOR_GREEN_SPLIT,
77            ANDROID_SENSOR_NEUTRAL_COLOR_POINT,
78            ANDROID_COLOR_CORRECTION_MODE,
79            ANDROID_COLOR_CORRECTION_TRANSFORM,
80            ANDROID_COLOR_CORRECTION_GAINS,
81     };
82     for (auto tag : tagsToRemove) {
83         res = resultMetadata.erase(tag);
84         if (res != OK) {
85             ALOGE("%s: Failed to remove tag %d for monochrome camera", __FUNCTION__, tag);
86             return res;
87         }
88     }
89 
90     // ANDROID_SENSOR_DYNAMIC_BLACK_LEVEL
91     camera_metadata_entry blEntry = resultMetadata.find(ANDROID_SENSOR_DYNAMIC_BLACK_LEVEL);
92     for (size_t i = 1; i < blEntry.count; i++) {
93         blEntry.data.f[i] = blEntry.data.f[0];
94     }
95 
96     // ANDROID_SENSOR_NOISE_PROFILE
97     camera_metadata_entry npEntry = resultMetadata.find(ANDROID_SENSOR_NOISE_PROFILE);
98     if (npEntry.count > 0 && npEntry.count % 2 == 0) {
99         double np[] = {npEntry.data.d[0], npEntry.data.d[1]};
100         res = resultMetadata.update(ANDROID_SENSOR_NOISE_PROFILE, np, 2);
101         if (res != OK) {
102              ALOGE("%s: Failed to update SENSOR_NOISE_PROFILE: %s (%d)",
103                     __FUNCTION__, strerror(-res), res);
104             return res;
105         }
106     }
107 
108     // ANDROID_STATISTICS_LENS_SHADING_MAP
109     camera_metadata_ro_entry lsSizeEntry = deviceInfo.find(ANDROID_LENS_INFO_SHADING_MAP_SIZE);
110     camera_metadata_entry lsEntry = resultMetadata.find(ANDROID_STATISTICS_LENS_SHADING_MAP);
111     if (lsSizeEntry.count == 2 && lsEntry.count > 0
112             && (int32_t)lsEntry.count == 4 * lsSizeEntry.data.i32[0] * lsSizeEntry.data.i32[1]) {
113         for (int32_t i = 0; i < lsSizeEntry.data.i32[0] * lsSizeEntry.data.i32[1]; i++) {
114             lsEntry.data.f[4*i+1] = lsEntry.data.f[4*i];
115             lsEntry.data.f[4*i+2] = lsEntry.data.f[4*i];
116             lsEntry.data.f[4*i+3] = lsEntry.data.f[4*i];
117         }
118     }
119 
120     // ANDROID_TONEMAP_CURVE_BLUE
121     // ANDROID_TONEMAP_CURVE_GREEN
122     // ANDROID_TONEMAP_CURVE_RED
123     camera_metadata_entry tcbEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_BLUE);
124     camera_metadata_entry tcgEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_GREEN);
125     camera_metadata_entry tcrEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_RED);
126     if (tcbEntry.count > 0
127             && tcbEntry.count == tcgEntry.count
128             && tcbEntry.count == tcrEntry.count) {
129         for (size_t i = 0; i < tcbEntry.count; i++) {
130             tcbEntry.data.f[i] = tcrEntry.data.f[i];
131             tcgEntry.data.f[i] = tcrEntry.data.f[i];
132         }
133     }
134 
135     return res;
136 }
137 
fixupAutoframingTags(CameraMetadata & resultMetadata)138 status_t fixupAutoframingTags(CameraMetadata& resultMetadata) {
139     status_t res = OK;
140     camera_metadata_entry autoframingEntry =
141             resultMetadata.find(ANDROID_CONTROL_AUTOFRAMING);
142     if (autoframingEntry.count == 0) {
143         const uint8_t defaultAutoframingEntry = ANDROID_CONTROL_AUTOFRAMING_OFF;
144         res = resultMetadata.update(ANDROID_CONTROL_AUTOFRAMING, &defaultAutoframingEntry, 1);
145         if (res != OK) {
146             ALOGE("%s: Failed to update ANDROID_CONTROL_AUTOFRAMING: %s (%d)",
147                   __FUNCTION__, strerror(-res), res);
148             return res;
149         }
150     }
151 
152     camera_metadata_entry autoframingStateEntry =
153             resultMetadata.find(ANDROID_CONTROL_AUTOFRAMING_STATE);
154     if (autoframingStateEntry.count == 0) {
155         const uint8_t defaultAutoframingStateEntry = ANDROID_CONTROL_AUTOFRAMING_STATE_INACTIVE;
156         res = resultMetadata.update(ANDROID_CONTROL_AUTOFRAMING_STATE,
157                                     &defaultAutoframingStateEntry, 1);
158         if (res != OK) {
159             ALOGE("%s: Failed to update ANDROID_CONTROL_AUTOFRAMING_STATE: %s (%d)",
160                   __FUNCTION__, strerror(-res), res);
161             return res;
162         }
163     }
164 
165     return res;
166 }
167 
fixupManualFlashStrengthControlTags(CameraMetadata & resultMetadata)168 status_t fixupManualFlashStrengthControlTags(CameraMetadata& resultMetadata) {
169     status_t res = OK;
170     camera_metadata_entry strengthLevelEntry =
171             resultMetadata.find(ANDROID_FLASH_STRENGTH_LEVEL);
172     if (strengthLevelEntry.count == 0) {
173         const int32_t defaultStrengthLevelEntry = ANDROID_FLASH_STRENGTH_LEVEL;
174         res = resultMetadata.update(ANDROID_FLASH_STRENGTH_LEVEL, &defaultStrengthLevelEntry, 1);
175         if (res != OK) {
176             ALOGE("%s: Failed to update ANDROID_FLASH_STRENGTH_LEVEL: %s (%d)",
177                   __FUNCTION__, strerror(-res), res);
178             return res;
179         }
180     }
181 
182     return res;
183 }
184 
correctMeteringRegions(camera_metadata_t * meta)185 void correctMeteringRegions(camera_metadata_t *meta) {
186     if (meta == nullptr) return;
187 
188     uint32_t meteringRegionKeys[] = {
189             ANDROID_CONTROL_AE_REGIONS,
190             ANDROID_CONTROL_AWB_REGIONS,
191             ANDROID_CONTROL_AF_REGIONS };
192 
193     for (uint32_t key : meteringRegionKeys) {
194         camera_metadata_entry_t entry;
195         int res = find_camera_metadata_entry(meta, key, &entry);
196         if (res != OK) continue;
197 
198         for (size_t i = 0; i < entry.count; i += 5) {
199             if (entry.data.i32[0] > entry.data.i32[2]) {
200                 ALOGW("%s: Invalid metering region (%d): left: %d, right: %d",
201                         __FUNCTION__, key, entry.data.i32[0], entry.data.i32[2]);
202                 entry.data.i32[2] = entry.data.i32[0];
203             }
204             if (entry.data.i32[1] > entry.data.i32[3]) {
205                 ALOGW("%s: Invalid metering region (%d): top: %d, bottom: %d",
206                         __FUNCTION__, key, entry.data.i32[1], entry.data.i32[3]);
207                 entry.data.i32[3] = entry.data.i32[1];
208             }
209         }
210     }
211 }
212 
insertResultLocked(CaptureOutputStates & states,CaptureResult * result,uint32_t frameNumber)213 void insertResultLocked(CaptureOutputStates& states, CaptureResult *result, uint32_t frameNumber) {
214     if (result == nullptr) return;
215 
216     camera_metadata_t *meta = const_cast<camera_metadata_t *>(
217             result->mMetadata.getAndLock());
218     set_camera_metadata_vendor_id(meta, states.vendorTagId);
219     correctMeteringRegions(meta);
220     result->mMetadata.unlock(meta);
221 
222     if (result->mMetadata.update(ANDROID_REQUEST_FRAME_COUNT,
223             (int32_t*)&frameNumber, 1) != OK) {
224         SET_ERR("Failed to set frame number %d in metadata", frameNumber);
225         return;
226     }
227 
228     if (result->mMetadata.update(ANDROID_REQUEST_ID, &result->mResultExtras.requestId, 1) != OK) {
229         SET_ERR("Failed to set request ID in metadata for frame %d", frameNumber);
230         return;
231     }
232 
233     // Update vendor tag id for physical metadata
234     for (auto& physicalMetadata : result->mPhysicalMetadatas) {
235         auto &metadata =
236                 physicalMetadata.mCameraMetadataInfo.get<CameraMetadataInfo::metadata>();
237         camera_metadata_t *pmeta = const_cast<camera_metadata_t *>(metadata.getAndLock());
238         set_camera_metadata_vendor_id(pmeta, states.vendorTagId);
239         correctMeteringRegions(pmeta);
240         metadata.unlock(pmeta);
241     }
242 
243     // Valid result, insert into queue
244     std::list<CaptureResult>::iterator queuedResult =
245             states.resultQueue.insert(states.resultQueue.end(), CaptureResult(*result));
246     ALOGV("%s: result requestId = %" PRId32 ", frameNumber = %" PRId64
247            ", burstId = %" PRId32, __FUNCTION__,
248            queuedResult->mResultExtras.requestId,
249            queuedResult->mResultExtras.frameNumber,
250            queuedResult->mResultExtras.burstId);
251 
252     states.resultSignal.notify_one();
253 }
254 
255 
sendPartialCaptureResult(CaptureOutputStates & states,const camera_metadata_t * partialResult,const CaptureResultExtras & resultExtras,uint32_t frameNumber)256 void sendPartialCaptureResult(CaptureOutputStates& states,
257         const camera_metadata_t * partialResult,
258         const CaptureResultExtras &resultExtras, uint32_t frameNumber) {
259     ATRACE_CALL();
260     std::lock_guard<std::mutex> l(states.outputLock);
261 
262     CaptureResult captureResult;
263     captureResult.mResultExtras = resultExtras;
264     captureResult.mMetadata = partialResult;
265 
266     // Fix up result metadata for monochrome camera.
267     status_t res = fixupMonochromeTags(states, states.deviceInfo, captureResult.mMetadata);
268     if (res != OK) {
269         SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res);
270         return;
271     }
272 
273     // Update partial result by removing keys remapped by DistortionCorrection, ZoomRatio,
274     // and RotationAndCrop mappers.
275     std::set<uint32_t> keysToRemove;
276 
277     auto iter = states.distortionMappers.find(states.cameraId);
278     if (iter != states.distortionMappers.end()) {
279         const auto& remappedKeys = iter->second.getRemappedKeys();
280         keysToRemove.insert(remappedKeys.begin(), remappedKeys.end());
281     }
282 
283     const auto& remappedKeys = states.zoomRatioMappers[states.cameraId].getRemappedKeys();
284     keysToRemove.insert(remappedKeys.begin(), remappedKeys.end());
285 
286     auto mapper = states.rotateAndCropMappers.find(states.cameraId);
287     if (mapper != states.rotateAndCropMappers.end()) {
288         const auto& remappedKeys = mapper->second.getRemappedKeys();
289         keysToRemove.insert(remappedKeys.begin(), remappedKeys.end());
290     }
291 
292     for (uint32_t key : keysToRemove) {
293         captureResult.mMetadata.erase(key);
294     }
295 
296     // Send partial result
297     if (captureResult.mMetadata.entryCount() > 0) {
298         insertResultLocked(states, &captureResult, frameNumber);
299     }
300 }
301 
sendCaptureResult(CaptureOutputStates & states,CameraMetadata & pendingMetadata,CaptureResultExtras & resultExtras,CameraMetadata & collectedPartialResult,uint32_t frameNumber,bool reprocess,bool zslStillCapture,bool rotateAndCropAuto,const std::set<std::string> & cameraIdsWithZoom,bool useZoomRatio,const std::vector<PhysicalCaptureResultInfo> & physicalMetadatas)302 void sendCaptureResult(
303         CaptureOutputStates& states,
304         CameraMetadata &pendingMetadata,
305         CaptureResultExtras &resultExtras,
306         CameraMetadata &collectedPartialResult,
307         uint32_t frameNumber,
308         bool reprocess, bool zslStillCapture, bool rotateAndCropAuto,
309         const std::set<std::string>& cameraIdsWithZoom, bool useZoomRatio,
310         const std::vector<PhysicalCaptureResultInfo>& physicalMetadatas) {
311     ATRACE_CALL();
312     if (pendingMetadata.isEmpty())
313         return;
314 
315     std::lock_guard<std::mutex> l(states.outputLock);
316 
317     // TODO: need to track errors for tighter bounds on expected frame number
318     if (reprocess) {
319         if (frameNumber < states.nextReprocResultFrameNum) {
320             SET_ERR("Out-of-order reprocess capture result metadata submitted! "
321                 "(got frame number %d, expecting %d)",
322                 frameNumber, states.nextReprocResultFrameNum);
323             return;
324         }
325         states.nextReprocResultFrameNum = frameNumber + 1;
326     } else if (zslStillCapture) {
327         if (frameNumber < states.nextZslResultFrameNum) {
328             SET_ERR("Out-of-order ZSL still capture result metadata submitted! "
329                 "(got frame number %d, expecting %d)",
330                 frameNumber, states.nextZslResultFrameNum);
331             return;
332         }
333         states.nextZslResultFrameNum = frameNumber + 1;
334     } else {
335         if (frameNumber < states.nextResultFrameNum) {
336             SET_ERR("Out-of-order capture result metadata submitted! "
337                     "(got frame number %d, expecting %d)",
338                     frameNumber, states.nextResultFrameNum);
339             return;
340         }
341         states.nextResultFrameNum = frameNumber + 1;
342     }
343 
344     CaptureResult captureResult;
345     captureResult.mResultExtras = resultExtras;
346     captureResult.mMetadata = pendingMetadata;
347     captureResult.mPhysicalMetadatas = physicalMetadatas;
348 
349     // Append any previous partials to form a complete result
350     if (states.usePartialResult && !collectedPartialResult.isEmpty()) {
351         captureResult.mMetadata.append(collectedPartialResult);
352     }
353 
354     captureResult.mMetadata.sort();
355 
356     // Check that there's a timestamp in the result metadata
357     camera_metadata_entry timestamp = captureResult.mMetadata.find(ANDROID_SENSOR_TIMESTAMP);
358     if (timestamp.count == 0) {
359         SET_ERR("No timestamp provided by HAL for frame %d!",
360                 frameNumber);
361         return;
362     }
363     nsecs_t sensorTimestamp = timestamp.data.i64[0];
364 
365     for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
366         camera_metadata_entry timestamp =
367                 physicalMetadata.mCameraMetadataInfo.get<CameraMetadataInfo::metadata>().
368                         find(ANDROID_SENSOR_TIMESTAMP);
369         if (timestamp.count == 0) {
370             SET_ERR("No timestamp provided by HAL for physical camera %s frame %d!",
371                     physicalMetadata.mPhysicalCameraId.c_str(), frameNumber);
372             return;
373         }
374     }
375 
376     // Fix up some result metadata to account for HAL-level distortion correction
377     status_t res = OK;
378     auto iter = states.distortionMappers.find(states.cameraId);
379     if (iter != states.distortionMappers.end()) {
380         res = iter->second.correctCaptureResult(&captureResult.mMetadata);
381         if (res != OK) {
382             SET_ERR("Unable to correct capture result metadata for frame %d: %s (%d)",
383                     frameNumber, strerror(-res), res);
384             return;
385         }
386     }
387 
388     // Fix up result metadata to account for zoom ratio availabilities between
389     // HAL and app.
390     bool zoomRatioIs1 = cameraIdsWithZoom.find(states.cameraId) == cameraIdsWithZoom.end();
391     res = states.zoomRatioMappers[states.cameraId].updateCaptureResult(
392             &captureResult.mMetadata, useZoomRatio, zoomRatioIs1);
393     if (res != OK) {
394         SET_ERR("Failed to update capture result zoom ratio metadata for frame %d: %s (%d)",
395                 frameNumber, strerror(-res), res);
396         return;
397     }
398 
399     // Fix up result metadata to account for rotateAndCrop in AUTO mode
400     if (rotateAndCropAuto) {
401         auto mapper = states.rotateAndCropMappers.find(states.cameraId);
402         if (mapper != states.rotateAndCropMappers.end()) {
403             res = mapper->second.updateCaptureResult(
404                     &captureResult.mMetadata);
405             if (res != OK) {
406                 SET_ERR("Unable to correct capture result rotate-and-crop for frame %d: %s (%d)",
407                         frameNumber, strerror(-res), res);
408                 return;
409             }
410         }
411     }
412 
413     // Fix up manual flash strength control metadata
414     res = fixupManualFlashStrengthControlTags(captureResult.mMetadata);
415     if (res != OK) {
416         SET_ERR("Failed to set flash strength level defaults in result metadata: %s (%d)",
417                 strerror(-res), res);
418         return;
419     }
420     for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
421         res = fixupManualFlashStrengthControlTags(physicalMetadata.mCameraMetadataInfo.
422                 get<CameraMetadataInfo::metadata>());
423         if (res != OK) {
424             SET_ERR("Failed to set flash strength level defaults in physical result"
425                     " metadata: %s (%d)", strerror(-res), res);
426             return;
427         }
428     }
429 
430     // Fix up autoframing metadata
431     res = fixupAutoframingTags(captureResult.mMetadata);
432     if (res != OK) {
433         SET_ERR("Failed to set autoframing defaults in result metadata: %s (%d)",
434                 strerror(-res), res);
435         return;
436     }
437     for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
438         res = fixupAutoframingTags(physicalMetadata.mCameraMetadataInfo.
439                 get<CameraMetadataInfo::metadata>());
440         if (res != OK) {
441             SET_ERR("Failed to set autoframing defaults in physical result metadata: %s (%d)",
442                     strerror(-res), res);
443             return;
444         }
445     }
446 
447     for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
448         const std::string cameraId = physicalMetadata.mPhysicalCameraId;
449         auto mapper = states.distortionMappers.find(cameraId);
450         if (mapper != states.distortionMappers.end()) {
451             res = mapper->second.correctCaptureResult(
452                     &physicalMetadata.mCameraMetadataInfo.get<CameraMetadataInfo::metadata>());
453             if (res != OK) {
454                 SET_ERR("Unable to correct physical capture result metadata for frame %d: %s (%d)",
455                         frameNumber, strerror(-res), res);
456                 return;
457             }
458         }
459 
460         // Note: Physical camera continues to use SCALER_CROP_REGION to reflect
461         // zoom levels. Model this by treating app-set ZOOM_RATIO as 1x.
462         res = states.zoomRatioMappers[cameraId].updateCaptureResult(
463                 &physicalMetadata.mCameraMetadataInfo.get<CameraMetadataInfo::metadata>(),
464                 /*zoomMethodIsRatio*/false,
465                 /*zoomRatioIs1*/true);
466         if (res != OK) {
467             SET_ERR("Failed to update camera %s's physical zoom ratio metadata for "
468                     "frame %d: %s(%d)", cameraId.c_str(), frameNumber, strerror(-res), res);
469             return;
470         }
471     }
472 
473     // Fix up result metadata for monochrome camera.
474     res = fixupMonochromeTags(states, states.deviceInfo, captureResult.mMetadata);
475     if (res != OK) {
476         SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res);
477         return;
478     }
479     for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
480         const std::string &cameraId = physicalMetadata.mPhysicalCameraId;
481         res = fixupMonochromeTags(states,
482                 states.physicalDeviceInfoMap.at(cameraId),
483                 physicalMetadata.mCameraMetadataInfo.get<CameraMetadataInfo::metadata>());
484         if (res != OK) {
485             SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res);
486             return;
487         }
488     }
489 
490     std::unordered_map<std::string, CameraMetadata> monitoredPhysicalMetadata;
491     for (auto& m : physicalMetadatas) {
492         monitoredPhysicalMetadata.emplace(m.mPhysicalCameraId,
493                 CameraMetadata(m.mCameraMetadataInfo.get<CameraMetadataInfo::metadata>()));
494     }
495     states.tagMonitor.monitorMetadata(TagMonitor::RESULT,
496             frameNumber, sensorTimestamp, captureResult.mMetadata,
497             monitoredPhysicalMetadata);
498 
499     insertResultLocked(states, &captureResult, frameNumber);
500 }
501 
removeInFlightMapEntryLocked(CaptureOutputStates & states,int idx)502 void removeInFlightMapEntryLocked(CaptureOutputStates& states, int idx) {
503     ATRACE_CALL();
504     InFlightRequestMap& inflightMap = states.inflightMap;
505     nsecs_t duration = inflightMap.valueAt(idx).maxExpectedDuration;
506     inflightMap.removeItemsAt(idx, 1);
507 
508     states.inflightIntf.onInflightEntryRemovedLocked(duration);
509 }
510 
removeInFlightRequestIfReadyLocked(CaptureOutputStates & states,int idx,std::vector<BufferToReturn> * returnableBuffers)511 void removeInFlightRequestIfReadyLocked(CaptureOutputStates& states, int idx,
512         std::vector<BufferToReturn> *returnableBuffers) {
513     InFlightRequestMap& inflightMap = states.inflightMap;
514     const InFlightRequest &request = inflightMap.valueAt(idx);
515     const uint32_t frameNumber = inflightMap.keyAt(idx);
516     SessionStatsBuilder& sessionStatsBuilder = states.sessionStatsBuilder;
517 
518     nsecs_t sensorTimestamp = request.sensorTimestamp;
519     nsecs_t shutterTimestamp = request.shutterTimestamp;
520 
521     // Check if it's okay to remove the request from InFlightMap:
522     // In the case of a successful request:
523     //      all input and output buffers, all result metadata, shutter callback
524     //      arrived.
525     // In the case of an unsuccessful request:
526     //      all input and output buffers, as well as request/result error notifications, arrived.
527     if (request.numBuffersLeft == 0 &&
528             (request.skipResultMetadata ||
529             (request.haveResultMetadata && shutterTimestamp != 0))) {
530         if (request.stillCapture) {
531             ATRACE_ASYNC_END("still capture", frameNumber);
532         }
533 
534         ATRACE_ASYNC_END("frame capture", frameNumber);
535 
536         // Validation check - if sensor timestamp matches shutter timestamp in the
537         // case of request having callback.
538         if (request.hasCallback && request.requestStatus == OK &&
539                 sensorTimestamp != shutterTimestamp) {
540             SET_ERR("sensor timestamp (%" PRId64
541                 ") for frame %d doesn't match shutter timestamp (%" PRId64 ")",
542                 sensorTimestamp, frameNumber, shutterTimestamp);
543         }
544 
545         // for an unsuccessful request, it may have pending output buffers to
546         // return.
547         assert(request.requestStatus != OK ||
548                request.pendingOutputBuffers.size() == 0);
549 
550         collectReturnableOutputBuffers(
551             states.useHalBufManager, states.halBufManagedStreamIds,
552             states.listener,
553             request.pendingOutputBuffers.array(),
554             request.pendingOutputBuffers.size(), /*timestamp*/0, /*readoutTimestamp*/0,
555             /*requested*/true, request.requestTimeNs, states.sessionStatsBuilder,
556             /*out*/ returnableBuffers,
557             /*timestampIncreasing*/true,
558             request.outputSurfaces, request.resultExtras,
559             request.errorBufStrategy, request.transform);
560 
561         // Note down the just completed frame number
562         if (request.hasInputBuffer) {
563             states.lastCompletedReprocessFrameNumber = frameNumber;
564         } else if (request.zslCapture && request.stillCapture) {
565             states.lastCompletedZslFrameNumber = frameNumber;
566         } else {
567             states.lastCompletedRegularFrameNumber = frameNumber;
568         }
569 
570         sessionStatsBuilder.incResultCounter(request.skipResultMetadata);
571 
572         removeInFlightMapEntryLocked(states, idx);
573         ALOGVV("%s: removed frame %d from InFlightMap", __FUNCTION__, frameNumber);
574     }
575 
576     states.inflightIntf.checkInflightMapLengthLocked();
577 }
578 
579 // Erase the subset of physicalCameraIds that contains id
erasePhysicalCameraIdSet(std::set<std::set<std::string>> & physicalCameraIds,const std::string & id)580 bool erasePhysicalCameraIdSet(
581         std::set<std::set<std::string>>& physicalCameraIds, const std::string& id) {
582     bool found = false;
583     for (auto iter = physicalCameraIds.begin(); iter != physicalCameraIds.end(); iter++) {
584         if (iter->count(id) == 1) {
585             physicalCameraIds.erase(iter);
586             found = true;
587             break;
588         }
589     }
590     return found;
591 }
592 
getCameraIdsWithZoomLocked(const InFlightRequestMap & inflightMap,const CameraMetadata & metadata,const std::set<std::string> & cameraIdsWithZoom)593 const std::set<std::string>& getCameraIdsWithZoomLocked(
594         const InFlightRequestMap& inflightMap, const CameraMetadata& metadata,
595         const std::set<std::string>& cameraIdsWithZoom) {
596     camera_metadata_ro_entry overrideEntry =
597             metadata.find(ANDROID_CONTROL_SETTINGS_OVERRIDE);
598     camera_metadata_ro_entry frameNumberEntry =
599             metadata.find(ANDROID_CONTROL_SETTINGS_OVERRIDING_FRAME_NUMBER);
600     if (overrideEntry.count != 1
601             || overrideEntry.data.i32[0] != ANDROID_CONTROL_SETTINGS_OVERRIDE_ZOOM
602             || frameNumberEntry.count != 1) {
603         // No valid overriding frame number, skip
604         return cameraIdsWithZoom;
605     }
606 
607     uint32_t overridingFrameNumber = frameNumberEntry.data.i32[0];
608     ssize_t idx = inflightMap.indexOfKey(overridingFrameNumber);
609     if (idx < 0) {
610         ALOGE("%s: Failed to find pending request #%d in inflight map",
611                 __FUNCTION__, overridingFrameNumber);
612         return cameraIdsWithZoom;
613     }
614 
615     const InFlightRequest &r = inflightMap.valueFor(overridingFrameNumber);
616     return r.cameraIdsWithZoom;
617 }
618 
processCaptureResult(CaptureOutputStates & states,const camera_capture_result * result)619 void processCaptureResult(CaptureOutputStates& states, const camera_capture_result *result) {
620     ATRACE_CALL();
621 
622     status_t res;
623 
624     uint32_t frameNumber = result->frame_number;
625     if (result->result == NULL && result->num_output_buffers == 0 &&
626             result->input_buffer == NULL) {
627         SET_ERR("No result data provided by HAL for frame %d",
628                 frameNumber);
629         return;
630     }
631 
632     if (!states.usePartialResult &&
633             result->result != NULL &&
634             result->partial_result != 1) {
635         SET_ERR("Result is malformed for frame %d: partial_result %u must be 1"
636                 " if partial result is not supported",
637                 frameNumber, result->partial_result);
638         return;
639     }
640 
641     bool isPartialResult = false;
642     CameraMetadata collectedPartialResult;
643     bool hasInputBufferInRequest = false;
644 
645     // Get shutter timestamp and resultExtras from list of in-flight requests,
646     // where it was added by the shutter notification for this frame. If the
647     // shutter timestamp isn't received yet, append the output buffers to the
648     // in-flight request and they will be returned when the shutter timestamp
649     // arrives. Update the in-flight status and remove the in-flight entry if
650     // all result data and shutter timestamp have been received.
651     std::vector<BufferToReturn> returnableBuffers{};
652     nsecs_t shutterTimestamp = 0;
653     {
654         std::lock_guard<std::mutex> l(states.inflightLock);
655         ssize_t idx = states.inflightMap.indexOfKey(frameNumber);
656         if (idx == NAME_NOT_FOUND) {
657             SET_ERR("Unknown frame number for capture result: %d",
658                     frameNumber);
659             return;
660         }
661         InFlightRequest &request = states.inflightMap.editValueAt(idx);
662         ALOGVV("%s: got InFlightRequest requestId = %" PRId32
663                 ", frameNumber = %" PRId64 ", burstId = %" PRId32
664                 ", partialResultCount = %d/%d, hasCallback = %d, num_output_buffers %d"
665                 ", usePartialResult = %d",
666                 __FUNCTION__, request.resultExtras.requestId,
667                 request.resultExtras.frameNumber, request.resultExtras.burstId,
668                 result->partial_result, states.numPartialResults,
669                 request.hasCallback, result->num_output_buffers,
670                 states.usePartialResult);
671         // Always update the partial count to the latest one if it's not 0
672         // (buffers only). When framework aggregates adjacent partial results
673         // into one, the latest partial count will be used.
674         if (result->partial_result != 0)
675             request.resultExtras.partialResultCount = result->partial_result;
676 
677         if (result->result != nullptr) {
678             camera_metadata_ro_entry entry;
679             auto ret = find_camera_metadata_ro_entry(result->result,
680                     ANDROID_LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID, &entry);
681             if ((ret == OK) && (entry.count > 0)) {
682                 std::string physicalId(reinterpret_cast<const char *>(entry.data.u8));
683                 if (!states.activePhysicalId.empty() && physicalId != states.activePhysicalId) {
684                     states.listener->notifyPhysicalCameraChange(physicalId);
685                 }
686                 states.activePhysicalId = physicalId;
687                 using hardware::ICameraService::ROTATION_OVERRIDE_NONE;
688                 if (!states.legacyClient &&
689                         states.rotationOverride == ROTATION_OVERRIDE_NONE) {
690                     auto deviceInfo = states.physicalDeviceInfoMap.find(physicalId);
691                     if (deviceInfo != states.physicalDeviceInfoMap.end()) {
692                         auto orientation = deviceInfo->second.find(ANDROID_SENSOR_ORIENTATION);
693                         if (orientation.count > 0) {
694                             int32_t transform;
695                             ret = CameraUtils::getRotationTransform(deviceInfo->second,
696                                     OutputConfiguration::MIRROR_MODE_AUTO,
697                                             /*transformInverseDisplay*/true, &transform);
698                             if (ret == OK) {
699                                 // It is possible for camera providers to return the capture
700                                 // results after the processed frames. In such scenario, we will
701                                 // not be able to set the output transformation before the frames
702                                 // return back to the consumer for the current capture request
703                                 // but we could still try and configure it for any future requests
704                                 // that are still in flight. The assumption is that the physical
705                                 // device id remains the same for the duration of the pending queue.
706                                 for (size_t i = 0; i < states.inflightMap.size(); i++) {
707                                     auto &r = states.inflightMap.editValueAt(i);
708                                     if (r.requestTimeNs >= request.requestTimeNs) {
709                                         r.transform = transform;
710                                     }
711                                 }
712                             } else {
713                                 ALOGE("%s: Failed to calculate current stream transformation: %s "
714                                         "(%d)", __FUNCTION__, strerror(-ret), ret);
715                             }
716                         } else {
717                             ALOGE("%s: Physical device orientation absent!", __FUNCTION__);
718                         }
719                     } else {
720                         ALOGE("%s: Physical device not found in device info map found!",
721                                 __FUNCTION__);
722                     }
723                 }
724             }
725         }
726 
727         // Check if this result carries only partial metadata
728         if (states.usePartialResult && result->result != NULL) {
729             if (result->partial_result > states.numPartialResults || result->partial_result < 1) {
730                 SET_ERR("Result is malformed for frame %d: partial_result %u must be  in"
731                         " the range of [1, %d] when metadata is included in the result",
732                         frameNumber, result->partial_result, states.numPartialResults);
733                 return;
734             }
735             isPartialResult = (result->partial_result < states.numPartialResults);
736             if (isPartialResult && result->num_physcam_metadata) {
737                 SET_ERR("Result is malformed for frame %d: partial_result not allowed for"
738                         " physical camera result", frameNumber);
739                 return;
740             }
741             if (isPartialResult) {
742                 request.collectedPartialResult.append(result->result);
743             }
744 
745             if (isPartialResult && request.hasCallback) {
746                 // Send partial capture result
747                 sendPartialCaptureResult(states, result->result, request.resultExtras,
748                         frameNumber);
749             }
750         }
751 
752         shutterTimestamp = request.shutterTimestamp;
753         hasInputBufferInRequest = request.hasInputBuffer;
754 
755         // Did we get the (final) result metadata for this capture?
756         if (result->result != NULL && !isPartialResult) {
757             if (request.physicalCameraIds.size() != result->num_physcam_metadata) {
758                 SET_ERR("Expected physical Camera metadata count %d not equal to actual count %d",
759                         request.physicalCameraIds.size(), result->num_physcam_metadata);
760                 return;
761             }
762             if (request.haveResultMetadata) {
763                 SET_ERR("Called multiple times with metadata for frame %d",
764                         frameNumber);
765                 return;
766             }
767             for (uint32_t i = 0; i < result->num_physcam_metadata; i++) {
768                 const std::string physicalId = result->physcam_ids[i];
769                 bool validPhysicalCameraMetadata =
770                         erasePhysicalCameraIdSet(request.physicalCameraIds, physicalId);
771                 if (!validPhysicalCameraMetadata) {
772                     SET_ERR("Unexpected total result for frame %d camera %s",
773                             frameNumber, physicalId.c_str());
774                     return;
775                 }
776             }
777             if (states.usePartialResult &&
778                     !request.collectedPartialResult.isEmpty()) {
779                 collectedPartialResult.acquire(
780                     request.collectedPartialResult);
781             }
782             request.haveResultMetadata = true;
783             request.errorBufStrategy = ERROR_BUF_RETURN_NOTIFY;
784         }
785 
786         uint32_t numBuffersReturned = result->num_output_buffers;
787         if (result->input_buffer != NULL) {
788             if (hasInputBufferInRequest) {
789                 numBuffersReturned += 1;
790             } else {
791                 ALOGW("%s: Input buffer should be NULL if there is no input"
792                         " buffer sent in the request",
793                         __FUNCTION__);
794             }
795         }
796         request.numBuffersLeft -= numBuffersReturned;
797         if (request.numBuffersLeft < 0) {
798             SET_ERR("Too many buffers returned for frame %d",
799                     frameNumber);
800             return;
801         }
802 
803         camera_metadata_ro_entry_t entry;
804         res = find_camera_metadata_ro_entry(result->result,
805                 ANDROID_SENSOR_TIMESTAMP, &entry);
806         if (res == OK && entry.count == 1) {
807             request.sensorTimestamp = entry.data.i64[0];
808         }
809 
810         // If shutter event isn't received yet, do not return the pending output
811         // buffers.
812         request.pendingOutputBuffers.appendArray(result->output_buffers,
813                 result->num_output_buffers);
814         if (shutterTimestamp != 0) {
815             collectAndRemovePendingOutputBuffers(
816                 states.useHalBufManager, states.halBufManagedStreamIds,
817                 states.listener,
818                 request, states.sessionStatsBuilder,
819                 /*out*/ &returnableBuffers);
820         }
821 
822         if (result->result != NULL && !isPartialResult) {
823             for (uint32_t i = 0; i < result->num_physcam_metadata; i++) {
824                 CameraMetadata physicalMetadata;
825                 physicalMetadata.append(result->physcam_metadata[i]);
826                 request.physicalMetadatas.push_back({result->physcam_ids[i],
827                         physicalMetadata});
828             }
829             if (shutterTimestamp == 0) {
830                 request.pendingMetadata = result->result;
831                 request.collectedPartialResult = collectedPartialResult;
832             } else if (request.hasCallback) {
833                 CameraMetadata metadata;
834                 metadata = result->result;
835                 auto cameraIdsWithZoom = getCameraIdsWithZoomLocked(
836                         states.inflightMap, metadata, request.cameraIdsWithZoom);
837                 sendCaptureResult(states, metadata, request.resultExtras,
838                     collectedPartialResult, frameNumber,
839                     hasInputBufferInRequest, request.zslCapture && request.stillCapture,
840                     request.rotateAndCropAuto, cameraIdsWithZoom, request.useZoomRatio,
841                     request.physicalMetadatas);
842             }
843         }
844         removeInFlightRequestIfReadyLocked(states, idx, &returnableBuffers);
845         if (!flags::return_buffers_outside_locks()) {
846             finishReturningOutputBuffers(returnableBuffers,
847                 states.listener, states.sessionStatsBuilder);
848         }
849     } // scope for states.inFlightLock
850 
851     if (flags::return_buffers_outside_locks()) {
852         finishReturningOutputBuffers(returnableBuffers,
853                 states.listener, states.sessionStatsBuilder);
854     }
855 
856     if (result->input_buffer != NULL) {
857         if (hasInputBufferInRequest) {
858             Camera3Stream *stream =
859                 Camera3Stream::cast(result->input_buffer->stream);
860             res = stream->returnInputBuffer(*(result->input_buffer));
861             // Note: stream may be deallocated at this point, if this buffer was the
862             // last reference to it.
863             if (res != OK) {
864                 ALOGE("%s: RequestThread: Can't return input buffer for frame %d to"
865                       "  its stream:%s (%d)",  __FUNCTION__,
866                       frameNumber, strerror(-res), res);
867             }
868         } else {
869             ALOGW("%s: Input buffer should be NULL if there is no input"
870                     " buffer sent in the request, skipping input buffer return.",
871                     __FUNCTION__);
872         }
873     }
874 }
875 
collectReturnableOutputBuffers(bool useHalBufManager,const std::set<int32_t> & halBufferManagedStreams,sp<NotificationListener> listener,const camera_stream_buffer_t * outputBuffers,size_t numBuffers,nsecs_t timestamp,nsecs_t readoutTimestamp,bool requested,nsecs_t requestTimeNs,SessionStatsBuilder & sessionStatsBuilder,std::vector<BufferToReturn> * returnableBuffers,bool timestampIncreasing,const SurfaceMap & outputSurfaces,const CaptureResultExtras & resultExtras,ERROR_BUF_STRATEGY errorBufStrategy,int32_t transform)876 void collectReturnableOutputBuffers(
877         bool useHalBufManager,
878         const std::set<int32_t> &halBufferManagedStreams,
879         sp<NotificationListener> listener,
880         const camera_stream_buffer_t *outputBuffers, size_t numBuffers,
881         nsecs_t timestamp, nsecs_t readoutTimestamp, bool requested,
882         nsecs_t requestTimeNs, SessionStatsBuilder& sessionStatsBuilder,
883         /*out*/ std::vector<BufferToReturn> *returnableBuffers,
884         bool timestampIncreasing, const SurfaceMap& outputSurfaces,
885         const CaptureResultExtras &resultExtras,
886         ERROR_BUF_STRATEGY errorBufStrategy, int32_t transform) {
887     for (size_t i = 0; i < numBuffers; i++)
888     {
889         Camera3StreamInterface *stream = Camera3Stream::cast(outputBuffers[i].stream);
890         int streamId = stream->getId();
891 
892         // Call notify(ERROR_BUFFER) if necessary.
893         if (outputBuffers[i].status == CAMERA_BUFFER_STATUS_ERROR &&
894                 errorBufStrategy == ERROR_BUF_RETURN_NOTIFY) {
895             if (listener != nullptr) {
896                 CaptureResultExtras extras = resultExtras;
897                 extras.errorStreamId = streamId;
898                 listener->notifyError(
899                         hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER,
900                         extras);
901             }
902         }
903 
904         if (outputBuffers[i].buffer == nullptr) {
905             if (!useHalBufManager &&
906                     !contains(halBufferManagedStreams, streamId)) {
907                 // With HAL buffer management API, HAL sometimes will have to return buffers that
908                 // has not got a output buffer handle filled yet. This is though illegal if HAL
909                 // buffer management API is not being used.
910                 ALOGE("%s: cannot return a null buffer!", __FUNCTION__);
911             } else {
912                 if (requested) {
913                     sessionStatsBuilder.incCounter(streamId, /*dropped*/true, 0);
914                 }
915             }
916             continue;
917         }
918 
919         const auto& it = outputSurfaces.find(streamId);
920 
921         // Do not return the buffer if the buffer status is error, and the error
922         // buffer strategy is CACHE.
923         if (outputBuffers[i].status != CAMERA_BUFFER_STATUS_ERROR ||
924                 errorBufStrategy != ERROR_BUF_CACHE) {
925             if (it != outputSurfaces.end()) {
926                 returnableBuffers->emplace_back(stream,
927                         outputBuffers[i], timestamp, readoutTimestamp, timestampIncreasing,
928                         it->second, resultExtras,
929                         transform, requested ? requestTimeNs : 0);
930             } else {
931                 returnableBuffers->emplace_back(stream,
932                         outputBuffers[i], timestamp, readoutTimestamp, timestampIncreasing,
933                         std::vector<size_t> (), resultExtras,
934                         transform, requested ? requestTimeNs : 0 );
935             }
936         }
937     }
938 }
939 
finishReturningOutputBuffers(const std::vector<BufferToReturn> & returnableBuffers,sp<NotificationListener> listener,SessionStatsBuilder & sessionStatsBuilder)940 void finishReturningOutputBuffers(const std::vector<BufferToReturn> &returnableBuffers,
941         sp<NotificationListener> listener, SessionStatsBuilder& sessionStatsBuilder) {
942     for (auto& b : returnableBuffers) {
943         const int streamId = b.stream->getId();
944 
945         status_t res = b.stream->returnBuffer(b.buffer, b.timestamp,
946                 b.readoutTimestamp, b.timestampIncreasing,
947                 b.surfaceIds, b.resultExtras.frameNumber, b.transform);
948 
949         // Note: stream may be deallocated at this point, if this buffer was
950         // the last reference to it.
951         bool dropped = false;
952         if (res == NO_INIT || res == DEAD_OBJECT) {
953             ALOGV("Can't return buffer to its stream: %s (%d)", strerror(-res), res);
954             sessionStatsBuilder.stopCounter(streamId);
955         } else if (res != OK) {
956             ALOGE("Can't return buffer to its stream: %s (%d)", strerror(-res), res);
957             dropped = true;
958         } else {
959             if (b.buffer.status == CAMERA_BUFFER_STATUS_ERROR || b.timestamp == 0) {
960                 dropped = true;
961             }
962         }
963         if (b.requestTimeNs > 0) {
964             nsecs_t bufferTimeNs = systemTime();
965             int32_t captureLatencyMs = ns2ms(bufferTimeNs - b.requestTimeNs);
966             sessionStatsBuilder.incCounter(streamId, dropped, captureLatencyMs);
967         }
968 
969         // Long processing consumers can cause returnBuffer timeout for shared stream
970         // If that happens, cancel the buffer and send a buffer error to client
971         if (b.surfaceIds.size() > 0 && res == TIMED_OUT &&
972                 b.buffer.status == CAMERA_BUFFER_STATUS_OK) {
973             // cancel the buffer
974             camera_stream_buffer_t sb = b.buffer;
975             sb.status = CAMERA_BUFFER_STATUS_ERROR;
976             b.stream->returnBuffer(sb, /*timestamp*/0, /*readoutTimestamp*/0,
977                     b.timestampIncreasing, std::vector<size_t> (),
978                     b.resultExtras.frameNumber, b.transform);
979 
980             if (listener != nullptr) {
981                 CaptureResultExtras extras = b.resultExtras;
982                 extras.errorStreamId = streamId;
983                 listener->notifyError(
984                         hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER,
985                         extras);
986             }
987         }
988 
989     }
990 }
991 
collectAndRemovePendingOutputBuffers(bool useHalBufManager,const std::set<int32_t> & halBufferManagedStreams,sp<NotificationListener> listener,InFlightRequest & request,SessionStatsBuilder & sessionStatsBuilder,std::vector<BufferToReturn> * returnableBuffers)992 void collectAndRemovePendingOutputBuffers(bool useHalBufManager,
993         const std::set<int32_t> &halBufferManagedStreams,
994         sp<NotificationListener> listener, InFlightRequest& request,
995         SessionStatsBuilder& sessionStatsBuilder,
996         std::vector<BufferToReturn> *returnableBuffers) {
997     bool timestampIncreasing =
998             !((request.zslCapture && request.stillCapture) || request.hasInputBuffer);
999     nsecs_t readoutTimestamp = request.resultExtras.hasReadoutTimestamp ?
1000             request.resultExtras.readoutTimestamp : 0;
1001     collectReturnableOutputBuffers(useHalBufManager, halBufferManagedStreams, listener,
1002             request.pendingOutputBuffers.array(),
1003             request.pendingOutputBuffers.size(),
1004             request.shutterTimestamp, readoutTimestamp,
1005             /*requested*/true, request.requestTimeNs, sessionStatsBuilder,
1006             /*out*/ returnableBuffers,
1007             timestampIncreasing,
1008             request.outputSurfaces, request.resultExtras,
1009             request.errorBufStrategy, request.transform);
1010 
1011     // Remove error buffers that are not cached.
1012     for (auto iter = request.pendingOutputBuffers.begin();
1013             iter != request.pendingOutputBuffers.end(); ) {
1014         if (request.errorBufStrategy != ERROR_BUF_CACHE ||
1015                 iter->status != CAMERA_BUFFER_STATUS_ERROR) {
1016             iter = request.pendingOutputBuffers.erase(iter);
1017         } else {
1018             iter++;
1019         }
1020     }
1021 }
1022 
notifyShutter(CaptureOutputStates & states,const camera_shutter_msg_t & msg)1023 void notifyShutter(CaptureOutputStates& states, const camera_shutter_msg_t &msg) {
1024     ATRACE_CALL();
1025     ssize_t idx;
1026 
1027     std::vector<BufferToReturn> returnableBuffers{};
1028     CaptureResultExtras pendingNotificationResultExtras{};
1029 
1030     // Set timestamp for the request in the in-flight tracking
1031     // and get the request ID to send upstream
1032     {
1033         std::lock_guard<std::mutex> l(states.inflightLock);
1034         InFlightRequestMap& inflightMap = states.inflightMap;
1035         idx = inflightMap.indexOfKey(msg.frame_number);
1036         if (idx >= 0) {
1037             InFlightRequest &r = inflightMap.editValueAt(idx);
1038 
1039             // Verify ordering of shutter notifications
1040             {
1041                 std::lock_guard<std::mutex> l(states.outputLock);
1042                 // TODO: need to track errors for tighter bounds on expected frame number.
1043                 if (r.hasInputBuffer) {
1044                     if (msg.frame_number < states.nextReprocShutterFrameNum) {
1045                         SET_ERR("Reprocess shutter notification out-of-order. Expected "
1046                                 "notification for frame %d, got frame %d",
1047                                 states.nextReprocShutterFrameNum, msg.frame_number);
1048                         return;
1049                     }
1050                     states.nextReprocShutterFrameNum = msg.frame_number + 1;
1051                 } else if (r.zslCapture && r.stillCapture) {
1052                     if (msg.frame_number < states.nextZslShutterFrameNum) {
1053                         SET_ERR("ZSL still capture shutter notification out-of-order. Expected "
1054                                 "notification for frame %d, got frame %d",
1055                                 states.nextZslShutterFrameNum, msg.frame_number);
1056                         return;
1057                     }
1058                     states.nextZslShutterFrameNum = msg.frame_number + 1;
1059                 } else {
1060                     if (msg.frame_number < states.nextShutterFrameNum) {
1061                         SET_ERR("Shutter notification out-of-order. Expected "
1062                                 "notification for frame %d, got frame %d",
1063                                 states.nextShutterFrameNum, msg.frame_number);
1064                         return;
1065                     }
1066                     states.nextShutterFrameNum = msg.frame_number + 1;
1067                 }
1068             }
1069 
1070             r.shutterTimestamp = msg.timestamp;
1071             if (msg.readout_timestamp_valid) {
1072                 r.resultExtras.hasReadoutTimestamp = true;
1073                 r.resultExtras.readoutTimestamp = msg.readout_timestamp;
1074             }
1075             if (r.minExpectedDuration != states.minFrameDuration ||
1076                     r.isFixedFps != states.isFixedFps) {
1077                 for (size_t i = 0; i < states.outputStreams.size(); i++) {
1078                     auto outputStream = states.outputStreams[i];
1079                     outputStream->onMinDurationChanged(r.minExpectedDuration, r.isFixedFps);
1080                 }
1081                 states.minFrameDuration = r.minExpectedDuration;
1082                 states.isFixedFps = r.isFixedFps;
1083             }
1084             if (r.hasCallback) {
1085                 ALOGVV("Camera %s: %s: Shutter fired for frame %d (id %d) at %" PRId64,
1086                     states.cameraId.c_str(), __FUNCTION__,
1087                     msg.frame_number, r.resultExtras.requestId, msg.timestamp);
1088                 // Call listener, if any
1089                 if (states.listener != nullptr) {
1090                     r.resultExtras.lastCompletedRegularFrameNumber =
1091                             states.lastCompletedRegularFrameNumber;
1092                     r.resultExtras.lastCompletedReprocessFrameNumber =
1093                             states.lastCompletedReprocessFrameNumber;
1094                     r.resultExtras.lastCompletedZslFrameNumber =
1095                             states.lastCompletedZslFrameNumber;
1096                     if (flags::return_buffers_outside_locks()) {
1097                         pendingNotificationResultExtras = r.resultExtras;
1098                     } else {
1099                         states.listener->notifyShutter(r.resultExtras, msg.timestamp);
1100                     }
1101                 }
1102                 // send pending result and buffers; this queues them up for delivery later
1103                 const auto& cameraIdsWithZoom = getCameraIdsWithZoomLocked(
1104                         inflightMap, r.pendingMetadata, r.cameraIdsWithZoom);
1105                 sendCaptureResult(states,
1106                     r.pendingMetadata, r.resultExtras,
1107                     r.collectedPartialResult, msg.frame_number,
1108                     r.hasInputBuffer, r.zslCapture && r.stillCapture,
1109                     r.rotateAndCropAuto, cameraIdsWithZoom, r.useZoomRatio,
1110                     r.physicalMetadatas);
1111             }
1112             collectAndRemovePendingOutputBuffers(
1113                     states.useHalBufManager, states.halBufManagedStreamIds,
1114                     states.listener, r, states.sessionStatsBuilder, &returnableBuffers);
1115 
1116             if (!flags::return_buffers_outside_locks()) {
1117                 finishReturningOutputBuffers(returnableBuffers,
1118                         states.listener, states.sessionStatsBuilder);
1119             }
1120 
1121             removeInFlightRequestIfReadyLocked(states, idx, &returnableBuffers);
1122 
1123         }
1124     }
1125     if (idx < 0) {
1126         SET_ERR("Shutter notification for non-existent frame number %d",
1127                 msg.frame_number);
1128     }
1129     // Call notifyShutter outside of in-flight mutex
1130     if (flags::return_buffers_outside_locks() && pendingNotificationResultExtras.isValid()) {
1131         states.listener->notifyShutter(pendingNotificationResultExtras, msg.timestamp);
1132     }
1133 
1134     // With no locks held, finish returning buffers to streams, which may take a while since
1135     // binder calls are involved
1136     if (flags::return_buffers_outside_locks()) {
1137         finishReturningOutputBuffers(returnableBuffers,
1138                 states.listener, states.sessionStatsBuilder);
1139     }
1140 
1141 }
1142 
notifyError(CaptureOutputStates & states,const camera_error_msg_t & msg)1143 void notifyError(CaptureOutputStates& states, const camera_error_msg_t &msg) {
1144     ATRACE_CALL();
1145     // Map camera HAL error codes to ICameraDeviceCallback error codes
1146     // Index into this with the HAL error code
1147     static const int32_t halErrorMap[CAMERA_MSG_NUM_ERRORS] = {
1148         // 0 = Unused error code
1149         hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR,
1150         // 1 = CAMERA_MSG_ERROR_DEVICE
1151         hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE,
1152         // 2 = CAMERA_MSG_ERROR_REQUEST
1153         hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST,
1154         // 3 = CAMERA_MSG_ERROR_RESULT
1155         hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT,
1156         // 4 = CAMERA_MSG_ERROR_BUFFER
1157         hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER
1158     };
1159 
1160     int32_t errorCode =
1161             ((msg.error_code >= 0) &&
1162                     (msg.error_code < CAMERA_MSG_NUM_ERRORS)) ?
1163             halErrorMap[msg.error_code] :
1164             hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR;
1165 
1166     int streamId = 0;
1167     std::string physicalCameraId;
1168     if (msg.error_stream != nullptr) {
1169         Camera3Stream *stream =
1170                 Camera3Stream::cast(msg.error_stream);
1171         streamId = stream->getId();
1172         physicalCameraId = stream->physicalCameraId();
1173     }
1174     ALOGV("Camera %s: %s: HAL error, frame %d, stream %d: %d",
1175             states.cameraId.c_str(), __FUNCTION__, msg.frame_number,
1176             streamId, msg.error_code);
1177 
1178     CaptureResultExtras resultExtras;
1179     switch (errorCode) {
1180         case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE:
1181             // SET_ERR calls into listener to notify application
1182             SET_ERR("Camera HAL reported serious device error");
1183             break;
1184         case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST:
1185         case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT:
1186         {
1187             std::vector<BufferToReturn> returnableBuffers{};
1188             {
1189                 std::lock_guard<std::mutex> l(states.inflightLock);
1190                 ssize_t idx = states.inflightMap.indexOfKey(msg.frame_number);
1191                 if (idx >= 0) {
1192                     InFlightRequest &r = states.inflightMap.editValueAt(idx);
1193                     r.requestStatus = msg.error_code;
1194                     resultExtras = r.resultExtras;
1195                     bool physicalDeviceResultError = false;
1196                     if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT ==
1197                             errorCode) {
1198                         if (physicalCameraId.size() > 0) {
1199                             bool validPhysicalCameraId =
1200                                     erasePhysicalCameraIdSet(r.physicalCameraIds, physicalCameraId);
1201                             if (!validPhysicalCameraId) {
1202                                 ALOGE("%s: Reported result failure for physical camera device: %s "
1203                                         " which is not part of the respective request!",
1204                                         __FUNCTION__, physicalCameraId.c_str());
1205                                 break;
1206                             }
1207                             resultExtras.errorPhysicalCameraId = physicalCameraId;
1208                             physicalDeviceResultError = true;
1209                         }
1210                     }
1211 
1212                     if (!physicalDeviceResultError) {
1213                         r.skipResultMetadata = true;
1214                         if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT
1215                                 == errorCode) {
1216                             r.errorBufStrategy = ERROR_BUF_RETURN_NOTIFY;
1217                         } else {
1218                             // errorCode is ERROR_CAMERA_REQUEST
1219                             r.errorBufStrategy = ERROR_BUF_RETURN;
1220                         }
1221 
1222                         // Check whether the buffers returned. If they returned,
1223                         // remove inflight request.
1224                         removeInFlightRequestIfReadyLocked(states, idx, &returnableBuffers);
1225                         if (!flags::return_buffers_outside_locks()) {
1226                             finishReturningOutputBuffers(returnableBuffers,
1227                                     states.listener, states.sessionStatsBuilder);
1228                         }
1229 
1230                     }
1231                 } else {
1232                     resultExtras.frameNumber = msg.frame_number;
1233                     ALOGE("Camera %s: %s: cannot find in-flight request on "
1234                             "frame %" PRId64 " error", states.cameraId.c_str(), __FUNCTION__,
1235                             resultExtras.frameNumber);
1236                 }
1237             }
1238 
1239             if (flags::return_buffers_outside_locks()) {
1240                 finishReturningOutputBuffers(returnableBuffers,
1241                         states.listener, states.sessionStatsBuilder);
1242             }
1243 
1244             resultExtras.errorStreamId = streamId;
1245             if (states.listener != nullptr) {
1246                 states.listener->notifyError(errorCode, resultExtras);
1247             } else {
1248                 ALOGE("Camera %s: %s: no listener available",
1249                         states.cameraId.c_str(), __FUNCTION__);
1250             }
1251             break;
1252         }
1253         case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER:
1254             // Do not depend on HAL ERROR_CAMERA_BUFFER to send buffer error
1255             // callback to the app. Rather, use STATUS_ERROR of image buffers.
1256             break;
1257         default:
1258             // SET_ERR calls notifyError
1259             SET_ERR("Unknown error message from HAL: %d", msg.error_code);
1260             break;
1261     }
1262 }
1263 
notify(CaptureOutputStates & states,const camera_notify_msg * msg)1264 void notify(CaptureOutputStates& states, const camera_notify_msg *msg) {
1265     switch (msg->type) {
1266         case CAMERA_MSG_ERROR: {
1267             notifyError(states, msg->message.error);
1268             break;
1269         }
1270         case CAMERA_MSG_SHUTTER: {
1271             notifyShutter(states, msg->message.shutter);
1272             break;
1273         }
1274         default:
1275             SET_ERR("Unknown notify message from HAL: %d",
1276                     msg->type);
1277     }
1278 }
1279 
flushInflightRequests(FlushInflightReqStates & states)1280 void flushInflightRequests(FlushInflightReqStates& states) {
1281     ATRACE_CALL();
1282     std::vector<BufferToReturn> returnableBuffers{};
1283     { // First return buffers cached in inFlightMap
1284         std::lock_guard<std::mutex> l(states.inflightLock);
1285         for (size_t idx = 0; idx < states.inflightMap.size(); idx++) {
1286             const InFlightRequest &request = states.inflightMap.valueAt(idx);
1287             collectReturnableOutputBuffers(
1288                 states.useHalBufManager, states.halBufManagedStreamIds,
1289                 states.listener,
1290                 request.pendingOutputBuffers.array(),
1291                 request.pendingOutputBuffers.size(), /*timestamp*/0, /*readoutTimestamp*/0,
1292                 /*requested*/true, request.requestTimeNs, states.sessionStatsBuilder,
1293                 /*out*/ &returnableBuffers,
1294                 /*timestampIncreasing*/true, request.outputSurfaces, request.resultExtras,
1295                 request.errorBufStrategy);
1296             if (!flags::return_buffers_outside_locks()) {
1297                 finishReturningOutputBuffers(returnableBuffers,
1298                         states.listener, states.sessionStatsBuilder);
1299             }
1300             ALOGW("%s: Frame %d |  Timestamp: %" PRId64 ", metadata"
1301                     " arrived: %s, buffers left: %d.\n", __FUNCTION__,
1302                     states.inflightMap.keyAt(idx), request.shutterTimestamp,
1303                     request.haveResultMetadata ? "true" : "false",
1304                     request.numBuffersLeft);
1305         }
1306 
1307         states.inflightMap.clear();
1308         states.inflightIntf.onInflightMapFlushedLocked();
1309     }
1310     if (flags::return_buffers_outside_locks()) {
1311         finishReturningOutputBuffers(returnableBuffers,
1312                 states.listener, states.sessionStatsBuilder);
1313     }
1314 
1315     // Then return all inflight buffers not returned by HAL
1316     std::vector<std::pair<int32_t, int32_t>> inflightKeys;
1317     states.flushBufferIntf.getInflightBufferKeys(&inflightKeys);
1318 
1319     // Inflight buffers for HAL buffer manager
1320     std::vector<uint64_t> inflightRequestBufferKeys;
1321     states.flushBufferIntf.getInflightRequestBufferKeys(&inflightRequestBufferKeys);
1322 
1323     // (streamId, frameNumber, buffer_handle_t*) tuple for all inflight buffers.
1324     // frameNumber will be -1 for buffers from HAL buffer manager
1325     std::vector<std::tuple<int32_t, int32_t, buffer_handle_t*>> inflightBuffers;
1326     inflightBuffers.reserve(inflightKeys.size() + inflightRequestBufferKeys.size());
1327 
1328     for (auto& pair : inflightKeys) {
1329         int32_t frameNumber = pair.first;
1330         int32_t streamId = pair.second;
1331         buffer_handle_t* buffer;
1332         status_t res = states.bufferRecordsIntf.popInflightBuffer(frameNumber, streamId, &buffer);
1333         if (res != OK) {
1334             ALOGE("%s: Frame %d: No in-flight buffer for stream %d",
1335                     __FUNCTION__, frameNumber, streamId);
1336             continue;
1337         }
1338         inflightBuffers.push_back(std::make_tuple(streamId, frameNumber, buffer));
1339     }
1340 
1341     for (auto& bufferId : inflightRequestBufferKeys) {
1342         int32_t streamId = -1;
1343         buffer_handle_t* buffer = nullptr;
1344         status_t res = states.bufferRecordsIntf.popInflightRequestBuffer(
1345                 bufferId, &buffer, &streamId);
1346         if (res != OK) {
1347             ALOGE("%s: cannot find in-flight buffer %" PRIu64, __FUNCTION__, bufferId);
1348             continue;
1349         }
1350         inflightBuffers.push_back(std::make_tuple(streamId, /*frameNumber*/-1, buffer));
1351     }
1352 
1353     std::vector<sp<Camera3StreamInterface>> streams = states.flushBufferIntf.getAllStreams();
1354 
1355     for (auto& tuple : inflightBuffers) {
1356         status_t res = OK;
1357         int32_t streamId = std::get<0>(tuple);
1358         int32_t frameNumber = std::get<1>(tuple);
1359         buffer_handle_t* buffer = std::get<2>(tuple);
1360 
1361         camera_stream_buffer_t streamBuffer;
1362         streamBuffer.buffer = buffer;
1363         streamBuffer.status = CAMERA_BUFFER_STATUS_ERROR;
1364         streamBuffer.acquire_fence = -1;
1365         streamBuffer.release_fence = -1;
1366 
1367         for (auto& stream : streams) {
1368             if (streamId == stream->getId()) {
1369                 // Return buffer to deleted stream
1370                 camera_stream* halStream = stream->asHalStream();
1371                 streamBuffer.stream = halStream;
1372                 switch (halStream->stream_type) {
1373                     case CAMERA_STREAM_OUTPUT:
1374                         res = stream->returnBuffer(streamBuffer, /*timestamp*/ 0,
1375                                 /*readoutTimestamp*/0, /*timestampIncreasing*/true,
1376                                 std::vector<size_t> (), frameNumber);
1377                         if (res != OK) {
1378                             ALOGE("%s: Can't return output buffer for frame %d to"
1379                                   " stream %d: %s (%d)",  __FUNCTION__,
1380                                   frameNumber, streamId, strerror(-res), res);
1381                         }
1382                         break;
1383                     case CAMERA_STREAM_INPUT:
1384                         res = stream->returnInputBuffer(streamBuffer);
1385                         if (res != OK) {
1386                             ALOGE("%s: Can't return input buffer for frame %d to"
1387                                   " stream %d: %s (%d)",  __FUNCTION__,
1388                                   frameNumber, streamId, strerror(-res), res);
1389                         }
1390                         break;
1391                     default: // Bi-direcitonal stream is deprecated
1392                         ALOGE("%s: stream %d has unknown stream type %d",
1393                                 __FUNCTION__, streamId, halStream->stream_type);
1394                         break;
1395                 }
1396                 break;
1397             }
1398         }
1399     }
1400 }
1401 
1402 } // camera3
1403 } // namespace android
1404