man/io_uring_prep_recv: expand on how to handle bundles

examples/proxy.c

@@ -183,6 +183,7 @@ struct conn_buf_ring {
 	struct io_uring_buf_ring *br;
 	void *buf;
 	int bgid;
+	uint16_t recv_head;  // used to track the buffer position in completions. 
 };
 
 struct conn {
@@ -392,6 +393,7 @@ static int setup_recv_ring(struct io_uring *ring, struct conn *c)
 		ptr += buf_size;
 	}
 	io_uring_buf_ring_advance(cbr->br, nr_bufs);
+	cbr->recv_head = 0;
 	printf("%d: recv buffer ring bgid %d, bufs %d\n", c->tid, cbr->bgid, nr_bufs);
 	return 0;
 }
@@ -923,12 +925,12 @@ static int replenish_buffers(struct conn *c, int *bid, int bytes)
 
 	while (bytes) {
 		int this_len = replenish_buffer(cbr, *bid, nr_packets);
-
+		++cbr->recv_head;
 		if (this_len > bytes)
 			this_len = bytes;
 		bytes -= this_len;
 
-		*bid = (*bid + 1) & (nr_bufs - 1);
+		*bid = cbr->br->bufs[cbr->recv_head & br_mask].bid;
 		nr_packets++;
 	}
 
@@ -1197,7 +1199,7 @@ static int recv_bids(struct conn *c, struct conn_dir *cd, int *bid, int in_bytes
 		int this_bytes;
 		void *data;
 
-		buf = &in_cbr->br->bufs[*bid];
+		buf = &in_cbr->br->bufs[in_cbr->recv_head & br_mask];
 		data = (void *) (unsigned long) buf->addr;
 		this_bytes = buf->len;
 		if (this_bytes > in_bytes)
@@ -1210,8 +1212,8 @@ static int recv_bids(struct conn *c, struct conn_dir *cd, int *bid, int in_bytes
 						br_mask, nr_packets);
 		else
 			send_append(c, cd, data, *bid, this_bytes);
-
-		*bid = (*bid + 1) & (nr_bufs - 1);
+		++in_cbr->recv_head;
+		*bid = in_cbr->br->bufs[in_cbr->recv_head & br_mask].bid;
 		nr_packets++;
 	}
 
@@ -1237,7 +1239,7 @@ static int recv_mshot_msg(struct conn *c, struct conn_dir *cd, int *bid,
 		int this_bytes;
 		void *data;
 
-		buf = &in_cbr->br->bufs[*bid];
+		buf = &in_cbr->br->bufs[in_cbr->recv_head & br_mask];
 
 		/*
 		 * multishot recvmsg puts a header in front of the data - we
@@ -1266,8 +1268,8 @@ static int recv_mshot_msg(struct conn *c, struct conn_dir *cd, int *bid,
 						br_mask, nr_packets);
 		else
 			send_append(c, cd, data, *bid, this_bytes);
-
-		*bid = (*bid + 1) & (nr_bufs - 1);
+		++in_cbr->recv_head;
+		*bid = in_cbr->br->bufs[in_cbr->recv_head & br_mask].bid;
 		nr_packets++;
 	}
 

man/io_uring_prep_recv.3

@@ -94,12 +94,18 @@ field indicates the total number of bytes received, and the buffer ID returned
 in the CQE
 .I flags
 field indicates the first buffer in the receive operation. The application must
-iterate from the indicated initial buffer ID and until all
+process the indicated initial buffer ID and until all
 .I res
 bytes have been seen to know which is the last buffer in the receive operation.
-The buffer IDs consumed will be contiguous from the starting ID, in the order
-in which they were added to the buffer ring used. Receiving in bundles can
-improve performance when more than one chunk of data is available to receive,
+The buffers consumed will be contiguous from the initial buffer, in the order
+in which they appear in the buffer ring. The CQE struct does not contain
+the position of the buffer in the buffer ring, therefore in order to identify
+buffers contained by the bundle, it is advised to maintain the cached head
+index per buffer ring. This uint16_t index represents the position of the next
+buffer to be consumed within the ring. Upon completion of a receive operation,
+the cached head index should be incremented accordingly.
+Receiving in bundles can improve performance when more than one chunk of
+data is available to receive,
 by eliminating redundant round trips through the networking stack. Receive
 bundles may be used by both single shot and multishot receive operations. Note
 that, internally, bundles rely on the networking stack passing back how much