libcamera: mali-c55: Implement capture for memory-to-memory

Plumb in the MaliC55 pipeline handler support for capturing frames from memory using the CRU. Introduce a data flow which uses the CRU to feed the ISP through the IVC. In detail: - push incoming request to a pending queue until a buffer from the CRU is available - delay the call to ipa_->fillParams() to the CRU buffer ready event - once the IPA has computed parameters feed the ISP through the IVC with buffers from the CRU, params and statistics - Handle completion of in-flight requests: in m2m mode buffers are queued earlier to the ISP capture devices. If the camera is stopped these buffers are returned earlier in error state: complete the Request bypassing the IPA operations - As cancelled Requests might now be completed earlier then IPA events, modify the IPA event handler to ignore Requests that have been completed already Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com> Reviewed-by: Daniel Scally <dan.scally@ideasonboard.com> Signed-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com> Signed-off-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
2026-04-24 14:38:27 +01:00
parent e14c06c19a
commit 88c636b097
1 changed files with 174 additions and 38 deletions
@@ -21,6 +21,7 @@
 #include <libcamera/base/utils.h>

 #include <libcamera/camera.h>
+#include <libcamera/controls.h>
 #include <libcamera/formats.h>
 #include <libcamera/geometry.h>
 #include <libcamera/property_ids.h>
@@ -89,6 +90,7 @@ struct MaliC55FrameInfo {

 	FrameBuffer *paramBuffer;
 	FrameBuffer *statBuffer;
+	FrameBuffer *rawBuffer;

 	bool paramsDone;
 	bool statsDone;
@@ -692,6 +694,7 @@ public:
 	void imageBufferReady(FrameBuffer *buffer);
 	void paramsBufferReady(FrameBuffer *buffer);
 	void statsBufferReady(FrameBuffer *buffer);
+	void cruBufferReady(FrameBuffer *buffer);
 	void paramsComputed(unsigned int requestId, uint32_t bytesused);
 	void statsProcessed(unsigned int requestId, const ControlList &metadata);

@@ -740,7 +743,7 @@ private:

 	MaliC55FrameInfo *findFrameInfo(FrameBuffer *buffer);
 	MaliC55FrameInfo *findFrameInfo(Request *request);
-	void tryComplete(MaliC55FrameInfo *info);
+	void tryComplete(MaliC55FrameInfo *info, bool cancelled = false);

 	int configureRawStream(MaliC55CameraData *data,
 			       const StreamConfiguration &config,
@@ -749,6 +752,9 @@ private:
 				     const StreamConfiguration &config,
 				     V4L2SubdeviceFormat &subdevFormat);

+	MaliC55FrameInfo *prepareFrameInfo(Request *request);
+	void queueRequestToCru(MaliC55CameraData *data, Request *request);
+
 	void applyScalerCrop(Camera *camera, const ControlList &controls);

 	bool registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,
@@ -1221,6 +1227,13 @@ void PipelineHandlerMaliC55::freeBuffers(Camera *camera)
 	if (params_->releaseBuffers())
 		LOG(MaliC55, Error) << "Failed to release params buffers";

+	if (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {
+		if (ivc_->releaseBuffers())
+			LOG(MaliC55, Error) << "Failed to release input buffers";
+		if (mem->cru_->freeBuffers())
+			LOG(MaliC55, Error) << "Failed to release CRU buffers";
+	}
+
 	return;
 }

@@ -1244,6 +1257,12 @@ int PipelineHandlerMaliC55::allocateBuffers(Camera *camera)
 		}
 	};

+	if (std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {
+		ret = ivc_->importBuffers(kMaliC55BufferCount);
+		if (ret < 0)
+			return ret;
+	}
+
 	ret = stats_->allocateBuffers(kMaliC55BufferCount, &statsBuffers_);
 	if (ret < 0)
 		return ret;
@@ -1275,6 +1294,24 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control
 	if (ret)
 		return ret;

+	if (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {
+		ret = mem->cru_->start(kMaliC55BufferCount);
+		if (ret) {
+			LOG(MaliC55, Error)
+				<< "Failed to start CRU " << camera->id();
+			freeBuffers(camera);
+			return ret;
+		}
+
+		ret = ivc_->streamOn();
+		if (ret) {
+			LOG(MaliC55, Error)
+				<< "Failed to start IVC" << camera->id();
+			freeBuffers(camera);
+			return ret;
+		}
+	}
+
 	if (data->ipa_) {
 		ret = data->ipa_->start();
 		if (ret) {
@@ -1356,6 +1393,11 @@ void PipelineHandlerMaliC55::stopDevice(Camera *camera)

 	isp_->setFrameStartEnabled(false);

+	if (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {
+		ivc_->streamOff();
+		mem->cru_->stop();
+	}
+
 	for (MaliC55Pipe &pipe : pipes_) {
 		if (!pipe.stream)
 			continue;
@@ -1467,10 +1509,68 @@ void PipelineHandlerMaliC55::applyScalerCrop(Camera *camera,
 	}
 }

+MaliC55FrameInfo *PipelineHandlerMaliC55::prepareFrameInfo(Request *request)
+{
+	if (availableStatsBuffers_.empty()) {
+		LOG(MaliC55, Error) << "Stats buffer underrun";
+		return nullptr;
+	}
+
+	if (availableParamsBuffers_.empty()) {
+		LOG(MaliC55, Error) << "Params buffer underrun";
+		return nullptr;
+	}
+
+	MaliC55FrameInfo &frameInfo = frameInfoMap_[request->sequence()];
+	frameInfo.request = request;
+	frameInfo.statBuffer = availableStatsBuffers_.front();
+	availableStatsBuffers_.pop();
+	frameInfo.paramBuffer = availableParamsBuffers_.front();
+	availableParamsBuffers_.pop();
+
+	frameInfo.paramsDone = false;
+	frameInfo.statsDone = false;
+
+	return &frameInfo;
+}
+
+void PipelineHandlerMaliC55::queueRequestToCru(MaliC55CameraData *data,
+					       Request *request)
+{
+	auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_);
+	ASSERT(mem);
+
+	FrameBuffer *cruBuffer = mem->cru_->queueBuffer(request);
+	ASSERT(cruBuffer);
+
+	auto frameInfo = prepareFrameInfo(request);
+	ASSERT(frameInfo);
+
+	frameInfo->rawBuffer = cruBuffer;
+
+	for (auto &[stream, buffer] : request->buffers()) {
+		MaliC55Pipe *pipe = pipeFromStream(data, stream);
+
+		pipe->cap->queueBuffer(buffer);
+	}
+
+	data->ipa_->queueRequest(request->sequence(), request->controls());
+}
+
 int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)
 {
 	MaliC55CameraData *data = cameraData(camera);

+	/*
+	 * If we're in memory input mode, we need to queue the Request to the
+	 * CRU, otherwise we can just do everything immediately.
+	 */
+	if (std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {
+		queueRequestToCru(data, request);
+
+		return 0;
+	}
+
 	/* Do not run the IPA if the TPG is in use. */
 	if (!data->ipa_) {
 		MaliC55FrameInfo frameInfo;
@@ -1491,32 +1591,12 @@ int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)
 		return 0;
 	}

-	if (availableStatsBuffers_.empty()) {
-		LOG(MaliC55, Error) << "Stats buffer underrun";
-		return -ENOENT;
-	}
-
-	if (availableParamsBuffers_.empty()) {
-		LOG(MaliC55, Error) << "Params buffer underrun";
-		return -ENOENT;
-	}
-
-	MaliC55FrameInfo frameInfo;
-	frameInfo.request = request;
-
-	frameInfo.statBuffer = availableStatsBuffers_.front();
-	availableStatsBuffers_.pop();
-	frameInfo.paramBuffer = availableParamsBuffers_.front();
-	availableParamsBuffers_.pop();
-
-	frameInfo.paramsDone = false;
-	frameInfo.statsDone = false;
-
-	frameInfoMap_[request->sequence()] = frameInfo;
+	auto frameInfo = prepareFrameInfo(request);
+	ASSERT(frameInfo);

 	data->ipa_->queueRequest(request->sequence(), request->controls());
 	data->ipa_->fillParams(request->sequence(),
-			       frameInfo.paramBuffer->cookie());
+			       frameInfo->paramBuffer->cookie());

 	return 0;
 }
@@ -1535,18 +1615,21 @@ MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(FrameBuffer *buffer)
 {
 	for (auto &[sequence, info] : frameInfoMap_) {
 		if (info.paramBuffer == buffer ||
-		    info.statBuffer == buffer)
+		    info.statBuffer == buffer ||
+		    info.rawBuffer == buffer)
 			return &info;
 	}

 	return nullptr;
 }

-void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info)
+void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info, bool cancelled)
 {
-	if (!info->paramsDone)
-		return;
-	if (!info->statsDone)
+	/*
+	 * If the buffer has been cancelled, we complete the request without
+	 * waiting for the IPA.
+	 */
+	if (!cancelled && (!info->paramsDone || !info->statsDone))
 		return;

 	Request *request = info->request;
@@ -1570,13 +1653,15 @@ void PipelineHandlerMaliC55::imageBufferReady(FrameBuffer *buffer)
 	ASSERT(info);

 	if (completeBuffer(request, buffer))
-		tryComplete(info);
+		tryComplete(info,
+			    buffer->metadata().status == FrameMetadata::FrameCancelled);
 }

 void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)
 {
 	MaliC55FrameInfo *info = findFrameInfo(buffer);
-	ASSERT(info);
+	if (!info)
+		return;

 	info->paramsDone = true;

@@ -1586,7 +1671,8 @@ void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)
 void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)
 {
 	MaliC55FrameInfo *info = findFrameInfo(buffer);
-	ASSERT(info);
+	if (!info)
+		return;

 	Request *request = info->request;
 	MaliC55CameraData *data = cameraData(request->_d()->camera());
@@ -1597,32 +1683,79 @@ void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)
 				 sensorControls);
 }

+void PipelineHandlerMaliC55::cruBufferReady(FrameBuffer *buffer)
+{
+	/*
+	 * If the buffer has been cancelled, do not ask the IPA to prepare
+	 * parameters.
+	 *
+	 * The Request this cancelled buffer belongs to will be handled by
+	 * imageBufferReady() as we have queued buffers to the ISP capture
+	 * devices at the same time we have queued this buffer to the CRU
+	 * when running in m2m mode.
+	 */
+	if (buffer->metadata().status == FrameMetadata::FrameCancelled)
+		return;
+
+	MaliC55FrameInfo *info = findFrameInfo(buffer);
+	ASSERT(info);
+
+	Request *request = info->request;
+	request->_d()->metadata().set(controls::SensorTimestamp,
+				      buffer->metadata().timestamp);
+
+	MaliC55CameraData *data = cameraData(request->_d()->camera());
+	data->ipa_->fillParams(request->sequence(), info->paramBuffer->cookie());
+}
+
 void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId, uint32_t bytesused)
 {
-	MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];
+	auto it = frameInfoMap_.find(requestId);
+	if (it == frameInfoMap_.end())
+		return;
+
+	MaliC55FrameInfo &frameInfo = it->second;
 	Request *request = frameInfo.request;
+	if (!request)
+		return;
+
 	MaliC55CameraData *data = cameraData(request->_d()->camera());

 	/*
 	 * Queue buffers for stats and params, then queue buffers to the capture
-	 * video devices.
+	 * video devices if we're running in Inline mode or with the TPG.
+	 *
+	 * If we're running in M2M buffers have been queued to the capture
+	 * devices at queueRequestToCru() time and here we only have to queue
+	 * buffers to the IVC input to start a transfer.
 	 */

 	frameInfo.paramBuffer->_d()->metadata().planes()[0].bytesused = bytesused;
 	params_->queueBuffer(frameInfo.paramBuffer);
 	stats_->queueBuffer(frameInfo.statBuffer);

-	for (auto &[stream, buffer] : request->buffers()) {
-		MaliC55Pipe *pipe = pipeFromStream(data, stream);
+	if (!std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {
+		for (auto &[stream, buffer] : request->buffers()) {
+			MaliC55Pipe *pipe = pipeFromStream(data, stream);

-		pipe->cap->queueBuffer(buffer);
+			pipe->cap->queueBuffer(buffer);
+		}
+	} else {
+		ivc_->queueBuffer(frameInfo.rawBuffer);
+		frameInfo.rawBuffer = nullptr;
 	}
 }

 void PipelineHandlerMaliC55::statsProcessed(unsigned int requestId,
 					    const ControlList &metadata)
 {
-	MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];
+	auto it = frameInfoMap_.find(requestId);
+	if (it == frameInfoMap_.end())
+		return;
+
+	MaliC55FrameInfo &frameInfo = it->second;
+	if (!frameInfo.request)
+		return;

 	frameInfo.statsDone = true;
 	frameInfo.request->_d()->metadata().merge(metadata);
@@ -1784,6 +1917,9 @@ bool PipelineHandlerMaliC55::registerMemoryInputCamera(MediaLink *link)

 	ivc_->bufferReady.connect(mem->cru_.get(), &RZG2LCRU::returnBuffer);

+	V4L2VideoDevice *cruOutput = mem->cru_->output();
+	cruOutput->bufferReady.connect(this, &PipelineHandlerMaliC55::cruBufferReady);
+
 	return registerMaliCamera(std::move(data), sensor->device()->entity()->name());
 }