108 	/* The consumer counter must always be smaller than the producer. */  in ck_ring_valid()
112 	/* The producer may only be up to size slots ahead of consumer. */  in ck_ring_valid()
146 	unsigned int consumer, producer, delta;  in _ck_ring_enqueue_reserve_sp()  local
148 	consumer = ck_pr_load_uint(&ring->c_head);  in _ck_ring_enqueue_reserve_sp()
152 		*size = (producer - consumer) & mask;  in _ck_ring_enqueue_reserve_sp()
154 	if (CK_CC_UNLIKELY((delta & mask) == (consumer & mask)))  in _ck_ring_enqueue_reserve_sp()
181 	unsigned int consumer, producer, delta;  in _ck_ring_enqueue_sp()  local
183 	consumer = ck_pr_load_uint(&ring->c_head);  in _ck_ring_enqueue_sp()
187 		*size = (producer - consumer) & mask;  in _ck_ring_enqueue_sp()
189 	if (CK_CC_UNLIKELY((delta & mask) == (consumer & mask)))  in _ck_ring_enqueue_sp()
226 	unsigned int consumer, producer;  in _ck_ring_dequeue_sc()  local
228 	consumer = ring->c_head;  in _ck_ring_dequeue_sc()
231 	if (CK_CC_UNLIKELY(consumer == producer))  in _ck_ring_dequeue_sc()
240 	buffer = (const char *)buffer + size * (consumer & mask);  in _ck_ring_dequeue_sc()
244 	 * Make sure copy is completed with respect to consumer  in _ck_ring_dequeue_sc()
248 	ck_pr_store_uint(&ring->c_head, consumer + 1);  in _ck_ring_dequeue_sc()
260 	unsigned int producer, consumer, delta;  in _ck_ring_enqueue_reserve_mp()  local
266 		consumer = ck_pr_load_uint(&ring->c_head);  in _ck_ring_enqueue_reserve_mp()
270 		if (CK_CC_LIKELY((producer - consumer) < mask)) {  in _ck_ring_enqueue_reserve_mp()
283 					*size = (producer - consumer) & mask;  in _ck_ring_enqueue_reserve_mp()
294 		*size = (producer - consumer) & mask;  in _ck_ring_enqueue_reserve_mp()
319 	unsigned int producer, consumer, delta;  in _ck_ring_enqueue_mp()  local
327 		 * consumer.  in _ck_ring_enqueue_mp()
330 		consumer = ck_pr_load_uint(&ring->c_head);  in _ck_ring_enqueue_mp()
336 		 * less than consumer) and the buffer is definitely not full.  in _ck_ring_enqueue_mp()
338 		if (CK_CC_LIKELY((producer - consumer) < mask)) {  in _ck_ring_enqueue_mp()
392 		*size = (producer - consumer) & mask;  in _ck_ring_enqueue_mp()
419 	unsigned int consumer, producer;  in _ck_ring_trydequeue_mc()  local
421 	consumer = ck_pr_load_uint(&ring->c_head);  in _ck_ring_trydequeue_mc()
425 	if (CK_CC_UNLIKELY(consumer == producer))  in _ck_ring_trydequeue_mc()
430 	buffer = (const char *)buffer + size * (consumer & mask);  in _ck_ring_trydequeue_mc()
434 	return ck_pr_cas_uint(&ring->c_head, consumer, consumer + 1);  in _ck_ring_trydequeue_mc()
444 	unsigned int consumer, producer;  in _ck_ring_dequeue_mc()  local
446 	consumer = ck_pr_load_uint(&ring->c_head);  in _ck_ring_dequeue_mc()
453 		 * our latest consumer snapshot.  in _ck_ring_dequeue_mc()
458 		if (CK_CC_UNLIKELY(consumer == producer))  in _ck_ring_dequeue_mc()
463 		target = (const char *)buffer + ts * (consumer & mask);  in _ck_ring_dequeue_mc()
469 				      consumer,  in _ck_ring_dequeue_mc()
470 				      consumer + 1,  in _ck_ring_dequeue_mc()
471 				      &consumer) == false);  in _ck_ring_dequeue_mc()
479  * to one concurrent consumer and up to one concurrent producer.
977  * A single producer with one concurrent consumer.
1008  * concurrent consumer.