1 /* 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef OHOS_CAMERA_H_CAPTURE_SESSION_H 17 #define OHOS_CAMERA_H_CAPTURE_SESSION_H 18 #define EXPORT_API __attribute__((visibility("default"))) 19 20 #include <atomic> 21 #include <cstdint> 22 #include <functional> 23 #include <iostream> 24 #include <list> 25 #include <memory> 26 #include <mutex> 27 #include <refbase.h> 28 #include <unordered_map> 29 #include <unordered_set> 30 #include "camera_util.h" 31 #include "fixed_size_list.h" 32 #include "hcamera_device.h" 33 #include "hcapture_session_stub.h" 34 #include "hstream_capture.h" 35 #include "hstream_metadata.h" 36 #include "hstream_repeat.h" 37 #include "icapture_session.h" 38 #include "istream_common.h" 39 #include "camera_photo_proxy.h" 40 #include "moving_photo/moving_photo_surface_wrapper.h" 41 #include "surface.h" 42 #include "v1_0/istream_operator.h" 43 #include "v1_1/istream_operator.h" 44 #include "v1_2/istream_operator.h" 45 #include "v1_3/istream_operator_callback.h" 46 #include "hcamera_restore_param.h" 47 #include "iconsumer_surface.h" 48 #include "blocking_queue.h" 49 #include "audio_capturer.h" 50 #include "audio_info.h" 51 #include "avcodec_task_manager.h" 52 #include "moving_photo_video_cache.h" 53 #include "drain_manager.h" 54 #include "audio_capturer_session.h" 55 #include "safe_map.h" 56 namespace OHOS::Media { 57 class Picture; 58 } 59 namespace OHOS { 60 namespace CameraStandard { 61 using OHOS::HDI::Camera::V1_0::CaptureEndedInfo; 62 using OHOS::HDI::Camera::V1_0::CaptureErrorInfo; 63 using namespace AudioStandard; 64 using namespace std::chrono; 65 using namespace DeferredProcessing; 66 using namespace Media; 67 class PermissionStatusChangeCb; 68 class CameraUseStateChangeCb; 69 class DisplayRotationListener; 70 class CameraServerPhotoProxy; 71 72 static const int32_t STREAM_NOT_FOUNT = -1; 73 74 enum class CaptureSessionReleaseType : int32_t { 75 RELEASE_TYPE_CLIENT = 0, 76 RELEASE_TYPE_CLIENT_DIED, 77 RELEASE_TYPE_SECURE, 78 RELEASE_TYPE_OBJ_DIED, 79 }; 80 81 class StateMachine { 82 public: 83 explicit StateMachine(); 84 virtual ~StateMachine() = default; 85 bool CheckTransfer(CaptureSessionState targetState); 86 bool Transfer(CaptureSessionState targetState); 87 GetCurrentState()88 inline CaptureSessionState GetCurrentState() 89 { 90 std::lock_guard<std::recursive_mutex> lock(sessionStateLock_); 91 return currentState_; 92 } 93 StateGuard(const std::function<void(const CaptureSessionState)>& fun)94 inline void StateGuard(const std::function<void(const CaptureSessionState)>& fun) 95 { 96 std::lock_guard<std::recursive_mutex> lock(sessionStateLock_); 97 fun(currentState_); 98 } 99 100 private: 101 std::vector<CaptureSessionState> stateTransferMap_[static_cast<uint32_t>(CaptureSessionState::SESSION_STATE_MAX)]; 102 std::recursive_mutex sessionStateLock_; 103 CaptureSessionState currentState_ = CaptureSessionState::SESSION_INIT; 104 }; 105 106 class StreamContainer { 107 public: StreamContainer()108 StreamContainer() {}; 109 virtual ~StreamContainer() = default; 110 111 bool AddStream(sptr<HStreamCommon> stream); 112 bool RemoveStream(sptr<HStreamCommon> stream); 113 sptr<HStreamCommon> GetStream(int32_t streamId); 114 sptr<HStreamCommon> GetHdiStream(int32_t streamId); 115 void Clear(); 116 size_t Size(); 117 118 std::list<sptr<HStreamCommon>> GetStreams(const StreamType streamType); 119 std::list<sptr<HStreamCommon>> GetAllStreams(); 120 121 private: 122 std::mutex streamsLock_; 123 std::map<const StreamType, std::list<sptr<HStreamCommon>>> streams_; 124 }; 125 126 class StreamOperatorCallback : public OHOS::HDI::Camera::V1_3::IStreamOperatorCallback { 127 public: 128 StreamOperatorCallback() = default; 129 virtual ~StreamOperatorCallback() = default; 130 131 int32_t OnCaptureStarted(int32_t captureId, const std::vector<int32_t>& streamIds) override; 132 int32_t OnCaptureStarted_V1_2( 133 int32_t captureId, const std::vector<OHOS::HDI::Camera::V1_2::CaptureStartedInfo>& infos) override; 134 int32_t OnCaptureEnded(int32_t captureId, const std::vector<CaptureEndedInfo>& infos) override; 135 int32_t OnCaptureEndedExt( 136 int32_t captureId, const std::vector<OHOS::HDI::Camera::V1_3::CaptureEndedInfoExt>& infos) override; 137 int32_t OnCaptureError(int32_t captureId, const std::vector<CaptureErrorInfo>& infos) override; 138 int32_t OnFrameShutter(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 139 int32_t OnFrameShutterEnd(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 140 int32_t OnCaptureReady(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 141 int32_t OnResult(int32_t streamId, const std::vector<uint8_t>& result) override; 142 143 virtual const sptr<HStreamCommon> GetStreamByStreamID(int32_t streamId) = 0; 144 virtual const sptr<HStreamCommon> GetHdiStreamByStreamID(int32_t streamId) = 0; 145 virtual void StartMovingPhotoEncode(int32_t rotation, uint64_t timestamp, int32_t format) = 0; 146 147 private: 148 std::mutex cbMutex_; 149 }; 150 151 class SessionDrainImageCallback; 152 using MetaElementType = std::pair<int64_t, sptr<SurfaceBuffer>>; 153 class MovingPhotoListener : public MovingPhotoSurfaceWrapper::SurfaceBufferListener { 154 public: 155 explicit MovingPhotoListener(sptr<MovingPhotoSurfaceWrapper> surfaceWrapper, sptr<Surface> metaSurface, 156 shared_ptr<FixedSizeList<MetaElementType>> metaCache); 157 ~MovingPhotoListener() override; 158 void OnBufferArrival(sptr<SurfaceBuffer> buffer, int64_t timestamp, GraphicTransformType transform) override; 159 void DrainOutImage(sptr<SessionDrainImageCallback> drainImageCallback); 160 void RemoveDrainImageManager(sptr<SessionDrainImageCallback> drainImageCallback); 161 void StopDrainOut(); 162 void ClearCache(uint64_t timestamp); 163 void SetClearFlag(); 164 165 private: 166 sptr<MovingPhotoSurfaceWrapper> movingPhotoSurfaceWrapper_; 167 sptr<Surface> metaSurface_; 168 shared_ptr<FixedSizeList<MetaElementType>> metaCache_; 169 BlockingQueue<sptr<FrameRecord>> recorderBufferQueue_; 170 SafeMap<sptr<SessionDrainImageCallback>, sptr<DrainImageManager>> callbackMap_; 171 std::atomic<bool> isNeededClear_ { false }; 172 std::atomic<bool> isNeededPop_ { false }; 173 int64_t shutterTime_; 174 }; 175 176 class MovingPhotoMetaListener : public IBufferConsumerListener { 177 public: 178 explicit MovingPhotoMetaListener(sptr<Surface> surface, shared_ptr<FixedSizeList<MetaElementType>> metaCache); 179 ~MovingPhotoMetaListener(); 180 void OnBufferAvailable() override; 181 private: 182 sptr<Surface> surface_; 183 shared_ptr<FixedSizeList<MetaElementType>> metaCache_; 184 }; 185 186 class SessionDrainImageCallback : public DrainImageCallback { 187 public: 188 explicit SessionDrainImageCallback(std::vector<sptr<FrameRecord>>& frameCacheList, 189 wptr<MovingPhotoListener> listener, 190 wptr<MovingPhotoVideoCache> cache, 191 uint64_t timestamp, 192 int32_t rotation); 193 ~SessionDrainImageCallback(); 194 void OnDrainImage(sptr<FrameRecord> frame) override; 195 void OnDrainImageFinish(bool isFinished) override; 196 197 private: 198 std::mutex mutex_; 199 std::vector<sptr<FrameRecord>> frameCacheList_; 200 wptr<MovingPhotoListener> listener_; 201 wptr<MovingPhotoVideoCache> videoCache_; 202 uint64_t timestamp_; 203 int32_t rotation_; 204 }; 205 206 class CameraInfoDumper; 207 208 class EXPORT_API HCaptureSession : public HCaptureSessionStub, public StreamOperatorCallback { 209 public: 210 static sptr<HCaptureSession> NewInstance(const uint32_t callerToken, int32_t opMode); 211 HCaptureSession(); 212 explicit HCaptureSession(const uint32_t callingTokenId, int32_t opMode); 213 virtual ~HCaptureSession(); 214 215 int32_t BeginConfig() override; 216 int32_t CommitConfig() override; 217 218 int32_t CanAddInput(sptr<ICameraDeviceService> cameraDevice, bool& result) override; 219 int32_t AddInput(sptr<ICameraDeviceService> cameraDevice) override; 220 int32_t AddOutput(StreamType streamType, sptr<IStreamCommon> stream) override; 221 222 int32_t RemoveInput(sptr<ICameraDeviceService> cameraDevice) override; 223 int32_t RemoveOutput(StreamType streamType, sptr<IStreamCommon> stream) override; 224 225 int32_t Start() override; 226 int32_t Stop() override; 227 int32_t Release() override; 228 int32_t Release(CaptureSessionReleaseType type); 229 230 static void DestroyStubObjectForPid(pid_t pid); 231 int32_t SetCallback(sptr<ICaptureSessionCallback>& callback) override; 232 233 int32_t GetSessionState(CaptureSessionState& sessionState) override; 234 int32_t GetActiveColorSpace(ColorSpace& colorSpace) override; 235 int32_t SetColorSpace(ColorSpace colorSpace, ColorSpace captureColorSpace, bool isNeedUpdate) override; 236 bool QueryFpsAndZoomRatio(float& currentFps, float& currentZoomRatio); 237 bool QueryZoomPerformance(std::vector<float>& crossZoomAndTime, int32_t operationMode); 238 int32_t SetSmoothZoom( 239 int32_t smoothZoomType, int32_t operationMode, float targetZoomRatio, float& duration) override; 240 int32_t EnableMovingPhoto(bool isEnable) override; 241 pid_t GetPid(); 242 int32_t GetCurrentStreamInfos(std::vector<StreamInfo_V1_1>& streamInfos); 243 int32_t GetopMode(); 244 245 int32_t OperatePermissionCheck(uint32_t interfaceCode) override; 246 int32_t EnableMovingPhotoMirror(bool isMirror) override; 247 int32_t CreateMediaLibrary(sptr<CameraPhotoProxy>& photoProxy, 248 std::string& uri, int32_t& cameraShotType, std::string& burstKey, int64_t timestamp) override; 249 int32_t CreateMediaLibrary(std::unique_ptr<Media::Picture> picture, sptr<CameraPhotoProxy>& photoProxy, 250 std::string &uri, int32_t &cameraShotType, std::string& burstKey, int64_t timestamp) override; 251 void SetCameraPhotoProxyInfo(sptr<CameraServerPhotoProxy> cameraPhotoProxy, int32_t &cameraShotType, 252 bool &isBursting, std::string &burstKey); 253 const sptr<HStreamCommon> GetStreamByStreamID(int32_t streamId) override; 254 const sptr<HStreamCommon> GetHdiStreamByStreamID(int32_t streamId) override; 255 int32_t SetFeatureMode(int32_t featureMode) override; 256 void StartMovingPhotoEncode(int32_t rotation, uint64_t timestamp, int32_t format) override; 257 void StartRecord(uint64_t timestamp, int32_t rotation); 258 void GetOutputStatus(int32_t &status); 259 int32_t SetPreviewRotation(std::string &deviceClass) override; 260 261 void DumpSessionInfo(CameraInfoDumper& infoDumper); 262 static void DumpSessions(CameraInfoDumper& infoDumper); 263 static void DumpCameraSessionSummary(CameraInfoDumper& infoDumper); 264 265 private: 266 int32_t Initialize(const uint32_t callerToken, int32_t opMode); 267 string lastDisplayName_ = ""; 268 string lastBurstPrefix_ = ""; 269 int32_t saveIndex = 0; 270 volatile bool isMovingPhotoMirror_ = false; 271 volatile bool isSetMotionPhoto_ = false; 272 std::mutex livePhotoStreamLock_; // Guard livePhotoStreamRepeat_ 273 sptr<HStreamRepeat> livePhotoStreamRepeat_; SetCameraDevice(sptr<HCameraDevice> device)274 inline void SetCameraDevice(sptr<HCameraDevice> device) 275 { 276 std::lock_guard<std::mutex> lock(cameraDeviceLock_); 277 cameraDevice_ = device; 278 } 279 GetCameraDevice()280 inline const sptr<HCameraDevice> GetCameraDevice() 281 { 282 std::lock_guard<std::mutex> lock(cameraDeviceLock_); 283 return cameraDevice_; 284 } 285 string CreateDisplayName(const std::string& suffix); 286 string CreateBurstDisplayName(int32_t seqId); 287 int32_t ValidateSessionInputs(); 288 int32_t ValidateSessionOutputs(); 289 int32_t ValidateSession(); 290 int32_t AddOutputStream(sptr<HStreamCommon> stream); 291 int32_t RemoveOutputStream(sptr<HStreamCommon> stream); 292 int32_t LinkInputAndOutputs(); 293 int32_t UnlinkInputAndOutputs(); 294 295 void ReleaseStreams(); 296 void ClearSketchRepeatStream(); 297 void ExpandSketchRepeatStream(); 298 void ExpandMovingPhotoRepeatStream(); 299 void ClearMovingPhotoRepeatStream(); 300 void StopMovingPhoto(); 301 int32_t CreateMovingPhotoStreamRepeat(int32_t format, int32_t width, int32_t height, 302 sptr<OHOS::IBufferProducer> producer); 303 int32_t CheckIfColorSpaceMatchesFormat(ColorSpace colorSpace); 304 void CancelStreamsAndGetStreamInfos(std::vector<StreamInfo_V1_1>& streamInfos); 305 void RestartStreams(); 306 int32_t UpdateStreamInfos(); 307 void SetColorSpaceForStreams(); 308 309 void ProcessMetaZoomArray(std::vector<uint32_t>& zoomAndTimeArray, sptr<HCameraDevice>& cameraDevice); 310 void StartMovingPhotoStream(); 311 bool InitAudioCapture(); 312 bool StartAudioCapture(); 313 void ProcessAudioBuffer(); 314 void StartOnceRecord(uint64_t timestamp, int32_t rotation); 315 int32_t StartPreviewStream(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, 316 camera_position_enum_t cameraPosition); 317 void UpdateMuteSetting(bool muteMode, std::shared_ptr<OHOS::Camera::CameraMetadata> &settings); 318 void StartMovingPhoto(sptr<HStreamRepeat>& curStreamRepeat); 319 int32_t GetSensorOritation(); 320 std::string GetSessionState(); 321 322 void DynamicConfigStream(); 323 bool IsNeedDynamicConfig(); 324 void RegisterDisplayListener(sptr<HStreamRepeat> repeat); 325 void UnRegisterDisplayListener(sptr<HStreamRepeat> repeat); 326 StateMachine stateMachine_; 327 328 // Make sure device thread safe,set device by {SetCameraDevice}, get device by {GetCameraDevice} 329 std::mutex cameraDeviceLock_; 330 sptr<HCameraDevice> cameraDevice_; 331 332 StreamContainer streamContainer_; 333 334 pid_t pid_; 335 uid_t uid_; 336 uint32_t callerToken_; 337 int32_t opMode_; 338 int32_t featureMode_; 339 ColorSpace currColorSpace_ = ColorSpace::COLOR_SPACE_UNKNOWN; 340 ColorSpace currCaptureColorSpace_ = ColorSpace::COLOR_SPACE_UNKNOWN; 341 bool isSessionStarted_ = false; 342 bool enableStreamRotate_ = false; 343 bool isDynamicConfiged_ = false; 344 std::string deviceClass_ = "phone"; 345 std::mutex movingPhotoStatusLock_; // Guard movingPhotoStatus 346 sptr<MovingPhotoListener> livephotoListener_; 347 sptr<MovingPhotoMetaListener> livephotoMetaListener_; 348 sptr<AudioCapturerSession> audioCapturerSession_; 349 sptr<Surface> metaSurface_ = nullptr; 350 sptr<MovingPhotoVideoCache> videoCache_; 351 sptr<AvcodecTaskManager> taskManager_; 352 std::mutex displayListenerLock_; 353 sptr<DisplayRotationListener> displayListener_; 354 }; 355 356 357 } // namespace CameraStandard 358 } // namespace OHOS 359 #endif // OHOS_CAMERA_H_CAPTURE_SESSION_H 360