libcamera: pipeline: simple: Support usage of multiple streams

To extend the multi-stream support to runtime operation of the pipeline,
expand the converter queue to store multiple output buffers, and update
the request queuing and buffer completion handlers accordingly.

Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Tested-by: Phi-Bang Nguyen <pnguyen@baylibre.com>
Reviewed-by: Paul Elder <paul.elder@ideasonboard.com>
Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>

src/libcamera/pipeline/simple/simple.cpp

@@ -150,6 +150,11 @@ public:
 	int setupFormats(V4L2SubdeviceFormat *format,
 			 V4L2Subdevice::Whence whence);
 
+	unsigned int streamIndex(const Stream *stream) const
+	{
+		return stream - &streams_.front();
+	}
+
 	struct Entity {
 		MediaEntity *entity;
 		MediaLink *link;
@@ -173,7 +178,7 @@ public:
 
 	std::vector<std::unique_ptr<FrameBuffer>> converterBuffers_;
 	bool useConverter_;
-	std::queue<FrameBuffer *> converterQueue_;
+	std::queue<std::map<unsigned int, FrameBuffer *>> converterQueue_;
 };
 
 class SimpleCameraConfiguration : public CameraConfiguration
@@ -764,7 +769,8 @@ int SimplePipelineHandler::exportFrameBuffers(Camera *camera, Stream *stream,
 	 * whether the converter is used or not.
 	 */
 	if (data->useConverter_)
-		return converter_->exportBuffers(0, count, buffers);
+		return converter_->exportBuffers(data->streamIndex(stream),
+						 count, buffers);
 	else
 		return data->video_->exportBuffers(count, buffers);
 }
@@ -831,25 +837,29 @@ void SimplePipelineHandler::stop(Camera *camera)
 int SimplePipelineHandler::queueRequestDevice(Camera *camera, Request *request)
 {
 	SimpleCameraData *data = cameraData(camera);
-	Stream *stream = &data->streams_[0];
+	int ret;
 
-	FrameBuffer *buffer = request->findBuffer(stream);
-	if (!buffer) {
-		LOG(SimplePipeline, Error)
-			<< "Attempt to queue request with invalid stream";
-		return -ENOENT;
+	std::map<unsigned int, FrameBuffer *> buffers;
+
+	for (auto &[stream, buffer] : request->buffers()) {
+		/*
+		 * If conversion is needed, push the buffer to the converter
+		 * queue, it will be handed to the converter in the capture
+		 * completion handler.
+		 */
+		if (data->useConverter_) {
+			buffers.emplace(data->streamIndex(stream), buffer);
+		} else {
+			ret = data->video_->queueBuffer(buffer);
+			if (ret < 0)
+				return ret;
+		}
 	}
 
-	/*
-	 * If conversion is needed, push the buffer to the converter queue, it
-	 * will be handed to the converter in the capture completion handler.
-	 */
-	if (data->useConverter_) {
-		data->converterQueue_.push(buffer);
-		return 0;
-	}
+	if (data->useConverter_)
+		data->converterQueue_.push(std::move(buffers));
 
-	return data->video_->queueBuffer(buffer);
+	return 0;
 }
 
 /* -----------------------------------------------------------------------------
@@ -1021,24 +1031,35 @@ void SimplePipelineHandler::bufferReady(FrameBuffer *buffer)
 	 * point converting an erroneous buffer.
 	 */
 	if (buffer->metadata().status != FrameMetadata::FrameSuccess) {
-		if (data->useConverter_) {
-			/* Requeue the buffer for capture. */
-			data->video_->queueBuffer(buffer);
-
-			/*
-			 * Get the next user-facing buffer to complete the
-			 * request.
-			 */
-			if (data->converterQueue_.empty())
-				return;
-
-			buffer = data->converterQueue_.front();
-			data->converterQueue_.pop();
+		if (!data->useConverter_) {
+			/* No conversion, just complete the request. */
+			Request *request = buffer->request();
+			completeBuffer(request, buffer);
+			completeRequest(request);
+			return;
 		}
 
-		Request *request = buffer->request();
-		completeBuffer(request, buffer);
-		completeRequest(request);
+		/*
+		 * The converter is in use. Requeue the internal buffer for
+		 * capture (unless the stream is being stopped), and complete
+		 * the request with all the user-facing buffers.
+		 */
+		if (buffer->metadata().status != FrameMetadata::FrameCancelled)
+			data->video_->queueBuffer(buffer);
+
+		if (data->converterQueue_.empty())
+			return;
+
+		Request *request = nullptr;
+		for (auto &item : data->converterQueue_.front()) {
+			FrameBuffer *outputBuffer = item.second;
+			request = outputBuffer->request();
+			completeBuffer(request, outputBuffer);
+		}
+		data->converterQueue_.pop();
+
+		if (request)
+			completeRequest(request);
 		return;
 	}
 
@@ -1053,10 +1074,8 @@ void SimplePipelineHandler::bufferReady(FrameBuffer *buffer)
 			return;
 		}
 
-		FrameBuffer *output = data->converterQueue_.front();
+		converter_->queueBuffers(buffer, data->converterQueue_.front());
 		data->converterQueue_.pop();
-
-		converter_->queueBuffers(buffer, { { 0, output } });
 		return;
 	}
 
@@ -1079,10 +1098,10 @@ void SimplePipelineHandler::converterOutputDone(FrameBuffer *buffer)
 {
 	ASSERT(activeCamera_);
 
-	/* Complete the request. */
+	/* Complete the buffer and the request. */
 	Request *request = buffer->request();
-	completeBuffer(request, buffer);
-	completeRequest(request);
+	if (completeBuffer(request, buffer))
+		completeRequest(request);
 }
 
 REGISTER_PIPELINE_HANDLER(SimplePipelineHandler)