From 88c636b097d6402414a434a8b69e72963db26f18 Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Tue, 31 Mar 2026 17:23:54 +0200 Subject: [PATCH] libcamera: mali-c55: Implement capture for memory-to-memory Plumb in the MaliC55 pipeline handler support for capturing frames from memory using the CRU. Introduce a data flow which uses the CRU to feed the ISP through the IVC. In detail: - push incoming request to a pending queue until a buffer from the CRU is available - delay the call to ipa_->fillParams() to the CRU buffer ready event - once the IPA has computed parameters feed the ISP through the IVC with buffers from the CRU, params and statistics - Handle completion of in-flight requests: in m2m mode buffers are queued earlier to the ISP capture devices. If the camera is stopped these buffers are returned earlier in error state: complete the Request bypassing the IPA operations - As cancelled Requests might now be completed earlier then IPA events, modify the IPA event handler to ignore Requests that have been completed already Signed-off-by: Daniel Scally Reviewed-by: Daniel Scally Signed-off-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- src/libcamera/pipeline/mali-c55/mali-c55.cpp | 212 +++++++++++++++---- 1 file changed, 174 insertions(+), 38 deletions(-) diff --git a/src/libcamera/pipeline/mali-c55/mali-c55.cpp b/src/libcamera/pipeline/mali-c55/mali-c55.cpp index 2b0e3c84..903b8256 100644 --- a/src/libcamera/pipeline/mali-c55/mali-c55.cpp +++ b/src/libcamera/pipeline/mali-c55/mali-c55.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -89,6 +90,7 @@ struct MaliC55FrameInfo { FrameBuffer *paramBuffer; FrameBuffer *statBuffer; + FrameBuffer *rawBuffer; bool paramsDone; bool statsDone; @@ -692,6 +694,7 @@ public: void imageBufferReady(FrameBuffer *buffer); void paramsBufferReady(FrameBuffer *buffer); void statsBufferReady(FrameBuffer *buffer); + void cruBufferReady(FrameBuffer *buffer); void paramsComputed(unsigned int requestId, uint32_t bytesused); void statsProcessed(unsigned int requestId, const ControlList &metadata); @@ -740,7 +743,7 @@ private: MaliC55FrameInfo *findFrameInfo(FrameBuffer *buffer); MaliC55FrameInfo *findFrameInfo(Request *request); - void tryComplete(MaliC55FrameInfo *info); + void tryComplete(MaliC55FrameInfo *info, bool cancelled = false); int configureRawStream(MaliC55CameraData *data, const StreamConfiguration &config, @@ -749,6 +752,9 @@ private: const StreamConfiguration &config, V4L2SubdeviceFormat &subdevFormat); + MaliC55FrameInfo *prepareFrameInfo(Request *request); + void queueRequestToCru(MaliC55CameraData *data, Request *request); + void applyScalerCrop(Camera *camera, const ControlList &controls); bool registerMaliCamera(std::unique_ptr data, @@ -1221,6 +1227,13 @@ void PipelineHandlerMaliC55::freeBuffers(Camera *camera) if (params_->releaseBuffers()) LOG(MaliC55, Error) << "Failed to release params buffers"; + if (auto *mem = std::get_if(&data->input_)) { + if (ivc_->releaseBuffers()) + LOG(MaliC55, Error) << "Failed to release input buffers"; + if (mem->cru_->freeBuffers()) + LOG(MaliC55, Error) << "Failed to release CRU buffers"; + } + return; } @@ -1244,6 +1257,12 @@ int PipelineHandlerMaliC55::allocateBuffers(Camera *camera) } }; + if (std::holds_alternative(data->input_)) { + ret = ivc_->importBuffers(kMaliC55BufferCount); + if (ret < 0) + return ret; + } + ret = stats_->allocateBuffers(kMaliC55BufferCount, &statsBuffers_); if (ret < 0) return ret; @@ -1275,6 +1294,24 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control if (ret) return ret; + if (auto *mem = std::get_if(&data->input_)) { + ret = mem->cru_->start(kMaliC55BufferCount); + if (ret) { + LOG(MaliC55, Error) + << "Failed to start CRU " << camera->id(); + freeBuffers(camera); + return ret; + } + + ret = ivc_->streamOn(); + if (ret) { + LOG(MaliC55, Error) + << "Failed to start IVC" << camera->id(); + freeBuffers(camera); + return ret; + } + } + if (data->ipa_) { ret = data->ipa_->start(); if (ret) { @@ -1356,6 +1393,11 @@ void PipelineHandlerMaliC55::stopDevice(Camera *camera) isp_->setFrameStartEnabled(false); + if (auto *mem = std::get_if(&data->input_)) { + ivc_->streamOff(); + mem->cru_->stop(); + } + for (MaliC55Pipe &pipe : pipes_) { if (!pipe.stream) continue; @@ -1467,10 +1509,68 @@ void PipelineHandlerMaliC55::applyScalerCrop(Camera *camera, } } +MaliC55FrameInfo *PipelineHandlerMaliC55::prepareFrameInfo(Request *request) +{ + if (availableStatsBuffers_.empty()) { + LOG(MaliC55, Error) << "Stats buffer underrun"; + return nullptr; + } + + if (availableParamsBuffers_.empty()) { + LOG(MaliC55, Error) << "Params buffer underrun"; + return nullptr; + } + + MaliC55FrameInfo &frameInfo = frameInfoMap_[request->sequence()]; + frameInfo.request = request; + frameInfo.statBuffer = availableStatsBuffers_.front(); + availableStatsBuffers_.pop(); + frameInfo.paramBuffer = availableParamsBuffers_.front(); + availableParamsBuffers_.pop(); + + frameInfo.paramsDone = false; + frameInfo.statsDone = false; + + return &frameInfo; +} + +void PipelineHandlerMaliC55::queueRequestToCru(MaliC55CameraData *data, + Request *request) +{ + auto *mem = std::get_if(&data->input_); + ASSERT(mem); + + FrameBuffer *cruBuffer = mem->cru_->queueBuffer(request); + ASSERT(cruBuffer); + + auto frameInfo = prepareFrameInfo(request); + ASSERT(frameInfo); + + frameInfo->rawBuffer = cruBuffer; + + for (auto &[stream, buffer] : request->buffers()) { + MaliC55Pipe *pipe = pipeFromStream(data, stream); + + pipe->cap->queueBuffer(buffer); + } + + data->ipa_->queueRequest(request->sequence(), request->controls()); +} + int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request) { MaliC55CameraData *data = cameraData(camera); + /* + * If we're in memory input mode, we need to queue the Request to the + * CRU, otherwise we can just do everything immediately. + */ + if (std::holds_alternative(data->input_)) { + queueRequestToCru(data, request); + + return 0; + } + /* Do not run the IPA if the TPG is in use. */ if (!data->ipa_) { MaliC55FrameInfo frameInfo; @@ -1491,32 +1591,12 @@ int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request) return 0; } - if (availableStatsBuffers_.empty()) { - LOG(MaliC55, Error) << "Stats buffer underrun"; - return -ENOENT; - } - - if (availableParamsBuffers_.empty()) { - LOG(MaliC55, Error) << "Params buffer underrun"; - return -ENOENT; - } - - MaliC55FrameInfo frameInfo; - frameInfo.request = request; - - frameInfo.statBuffer = availableStatsBuffers_.front(); - availableStatsBuffers_.pop(); - frameInfo.paramBuffer = availableParamsBuffers_.front(); - availableParamsBuffers_.pop(); - - frameInfo.paramsDone = false; - frameInfo.statsDone = false; - - frameInfoMap_[request->sequence()] = frameInfo; + auto frameInfo = prepareFrameInfo(request); + ASSERT(frameInfo); data->ipa_->queueRequest(request->sequence(), request->controls()); data->ipa_->fillParams(request->sequence(), - frameInfo.paramBuffer->cookie()); + frameInfo->paramBuffer->cookie()); return 0; } @@ -1535,18 +1615,21 @@ MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(FrameBuffer *buffer) { for (auto &[sequence, info] : frameInfoMap_) { if (info.paramBuffer == buffer || - info.statBuffer == buffer) + info.statBuffer == buffer || + info.rawBuffer == buffer) return &info; } return nullptr; } -void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info) +void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info, bool cancelled) { - if (!info->paramsDone) - return; - if (!info->statsDone) + /* + * If the buffer has been cancelled, we complete the request without + * waiting for the IPA. + */ + if (!cancelled && (!info->paramsDone || !info->statsDone)) return; Request *request = info->request; @@ -1570,13 +1653,15 @@ void PipelineHandlerMaliC55::imageBufferReady(FrameBuffer *buffer) ASSERT(info); if (completeBuffer(request, buffer)) - tryComplete(info); + tryComplete(info, + buffer->metadata().status == FrameMetadata::FrameCancelled); } void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer) { MaliC55FrameInfo *info = findFrameInfo(buffer); - ASSERT(info); + if (!info) + return; info->paramsDone = true; @@ -1586,7 +1671,8 @@ void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer) void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer) { MaliC55FrameInfo *info = findFrameInfo(buffer); - ASSERT(info); + if (!info) + return; Request *request = info->request; MaliC55CameraData *data = cameraData(request->_d()->camera()); @@ -1597,32 +1683,79 @@ void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer) sensorControls); } +void PipelineHandlerMaliC55::cruBufferReady(FrameBuffer *buffer) +{ + /* + * If the buffer has been cancelled, do not ask the IPA to prepare + * parameters. + * + * The Request this cancelled buffer belongs to will be handled by + * imageBufferReady() as we have queued buffers to the ISP capture + * devices at the same time we have queued this buffer to the CRU + * when running in m2m mode. + */ + if (buffer->metadata().status == FrameMetadata::FrameCancelled) + return; + + MaliC55FrameInfo *info = findFrameInfo(buffer); + ASSERT(info); + + Request *request = info->request; + request->_d()->metadata().set(controls::SensorTimestamp, + buffer->metadata().timestamp); + + MaliC55CameraData *data = cameraData(request->_d()->camera()); + data->ipa_->fillParams(request->sequence(), info->paramBuffer->cookie()); +} + void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId, uint32_t bytesused) { - MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId]; + auto it = frameInfoMap_.find(requestId); + if (it == frameInfoMap_.end()) + return; + + MaliC55FrameInfo &frameInfo = it->second; Request *request = frameInfo.request; + if (!request) + return; + MaliC55CameraData *data = cameraData(request->_d()->camera()); /* * Queue buffers for stats and params, then queue buffers to the capture - * video devices. + * video devices if we're running in Inline mode or with the TPG. + * + * If we're running in M2M buffers have been queued to the capture + * devices at queueRequestToCru() time and here we only have to queue + * buffers to the IVC input to start a transfer. */ frameInfo.paramBuffer->_d()->metadata().planes()[0].bytesused = bytesused; params_->queueBuffer(frameInfo.paramBuffer); stats_->queueBuffer(frameInfo.statBuffer); - for (auto &[stream, buffer] : request->buffers()) { - MaliC55Pipe *pipe = pipeFromStream(data, stream); + if (!std::holds_alternative(data->input_)) { + for (auto &[stream, buffer] : request->buffers()) { + MaliC55Pipe *pipe = pipeFromStream(data, stream); - pipe->cap->queueBuffer(buffer); + pipe->cap->queueBuffer(buffer); + } + } else { + ivc_->queueBuffer(frameInfo.rawBuffer); + frameInfo.rawBuffer = nullptr; } } void PipelineHandlerMaliC55::statsProcessed(unsigned int requestId, const ControlList &metadata) { - MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId]; + auto it = frameInfoMap_.find(requestId); + if (it == frameInfoMap_.end()) + return; + + MaliC55FrameInfo &frameInfo = it->second; + if (!frameInfo.request) + return; frameInfo.statsDone = true; frameInfo.request->_d()->metadata().merge(metadata); @@ -1784,6 +1917,9 @@ bool PipelineHandlerMaliC55::registerMemoryInputCamera(MediaLink *link) ivc_->bufferReady.connect(mem->cru_.get(), &RZG2LCRU::returnBuffer); + V4L2VideoDevice *cruOutput = mem->cru_->output(); + cruOutput->bufferReady.connect(this, &PipelineHandlerMaliC55::cruBufferReady); + return registerMaliCamera(std::move(data), sensor->device()->entity()->name()); }