diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp
index 7822bf1e..76bdad40 100644
--- a/src/libcamera/v4l2_videodevice.cpp
+++ b/src/libcamera/v4l2_videodevice.cpp
@@ -1682,11 +1682,15 @@ int V4L2VideoDevice::queueBuffer(FrameBuffer *buffer)
 	 */
 	if (planes.size() < numV4l2Planes) {
 		LOG(V4L2, Error) << "Frame buffer has too few planes";
+		cache_->put(buf.index);
+
 		return -EINVAL;
 	}
 
 	if (planes.size() != numV4l2Planes && !buffer->_d()->isContiguous()) {
 		LOG(V4L2, Error) << "Device format requires contiguous buffer";
+		cache_->put(buf.index);
+
 		return -EINVAL;
 	}
 
@@ -1729,6 +1733,8 @@ int V4L2VideoDevice::queueBuffer(FrameBuffer *buffer)
 				if (i != planes.size() - 1 && bytesused != length) {
 					LOG(V4L2, Error)
 						<< "Holes in multi-planar buffer not supported";
+					cache_->put(buf.index);
+
 					return -EINVAL;
 				}
 			}
@@ -1778,6 +1784,8 @@ int V4L2VideoDevice::queueBuffer(FrameBuffer *buffer)
 		LOG(V4L2, Error)
 			<< "Failed to queue buffer " << buf.index << ": "
 			<< strerror(-ret);
+		cache_->put(buf.index);
+
 		return ret;
 	}