diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp
index 71cc7e89..d206244f 100644
--- a/src/libcamera/v4l2_videodevice.cpp
+++ b/src/libcamera/v4l2_videodevice.cpp
@@ -1668,6 +1668,8 @@ int V4L2VideoDevice::queueBuffer(FrameBuffer *buffer)
 	if (ret < 0)
 		return ret;
 
+	auto guard = utils::scope_exit{ [&]() { cache_->put(buf.index); } };
+
 	buf.index = ret;
 	buf.type = bufferType_;
 	buf.memory = memoryType_;
@@ -1683,15 +1685,11 @@ int V4L2VideoDevice::queueBuffer(FrameBuffer *buffer)
 	 */
 	if (planes.size() < numV4l2Planes) {
 		LOG(V4L2, Error) << "Frame buffer has too few planes";
-		cache_->put(buf.index);
-
 		return -EINVAL;
 	}
 
 	if (planes.size() != numV4l2Planes && !buffer->_d()->isContiguous()) {
 		LOG(V4L2, Error) << "Device format requires contiguous buffer";
-		cache_->put(buf.index);
-
 		return -EINVAL;
 	}
 
@@ -1734,8 +1732,6 @@ int V4L2VideoDevice::queueBuffer(FrameBuffer *buffer)
 				if (i != planes.size() - 1 && bytesused != length) {
 					LOG(V4L2, Error)
 						<< "Holes in multi-planar buffer not supported";
-					cache_->put(buf.index);
-
 					return -EINVAL;
 				}
 			}
@@ -1785,8 +1781,6 @@ int V4L2VideoDevice::queueBuffer(FrameBuffer *buffer)
 		LOG(V4L2, Error)
 			<< "Failed to queue buffer " << buf.index << ": "
 			<< strerror(-ret);
-		cache_->put(buf.index);
-
 		return ret;
 	}
 
@@ -1798,6 +1792,7 @@ int V4L2VideoDevice::queueBuffer(FrameBuffer *buffer)
 
 	queuedBuffers_[buf.index] = buffer;
 
+	guard.release();
 	return 0;
 }