2  * Copyright 2010-2015 Samy Al Bahra.
54 	struct ck_hp_fifo_entry *head;  member
63 	fifo->head = fifo->tail = stub;  in ck_hp_fifo_init()
64 	stub->next = NULL;  in ck_hp_fifo_init()
72 	*stub = fifo->head;  in ck_hp_fifo_deinit()
73 	fifo->head = fifo->tail = NULL;  in ck_hp_fifo_deinit()
85 	entry->value = value;  in ck_hp_fifo_enqueue_mpmc()
86 	entry->next = NULL;  in ck_hp_fifo_enqueue_mpmc()
90 		tail = ck_pr_load_ptr(&fifo->tail);  in ck_hp_fifo_enqueue_mpmc()
92 		if (tail != ck_pr_load_ptr(&fifo->tail))  in ck_hp_fifo_enqueue_mpmc()
95 		next = ck_pr_load_ptr(&tail->next);  in ck_hp_fifo_enqueue_mpmc()
97 			ck_pr_cas_ptr(&fifo->tail, tail, next);  in ck_hp_fifo_enqueue_mpmc()
99 		} else if (ck_pr_cas_ptr(&fifo->tail->next, next, entry) == true)  in ck_hp_fifo_enqueue_mpmc()
104 	ck_pr_cas_ptr(&fifo->tail, tail, entry);  in ck_hp_fifo_enqueue_mpmc()
116 	entry->value = value;  in ck_hp_fifo_tryenqueue_mpmc()
117 	entry->next = NULL;  in ck_hp_fifo_tryenqueue_mpmc()
120 	tail = ck_pr_load_ptr(&fifo->tail);  in ck_hp_fifo_tryenqueue_mpmc()
122 	if (tail != ck_pr_load_ptr(&fifo->tail))  in ck_hp_fifo_tryenqueue_mpmc()
125 	next = ck_pr_load_ptr(&tail->next);  in ck_hp_fifo_tryenqueue_mpmc()
127 		ck_pr_cas_ptr(&fifo->tail, tail, next);  in ck_hp_fifo_tryenqueue_mpmc()
129 	} else if (ck_pr_cas_ptr(&fifo->tail->next, next, entry) == false)  in ck_hp_fifo_tryenqueue_mpmc()
133 	ck_pr_cas_ptr(&fifo->tail, tail, entry);  in ck_hp_fifo_tryenqueue_mpmc()
142 	struct ck_hp_fifo_entry *head, *tail, *next;  in ck_hp_fifo_dequeue_mpmc()  local
145 		head = ck_pr_load_ptr(&fifo->head);  in ck_hp_fifo_dequeue_mpmc()
147 		tail = ck_pr_load_ptr(&fifo->tail);  in ck_hp_fifo_dequeue_mpmc()
148 		ck_hp_set_fence(record, 0, head);  in ck_hp_fifo_dequeue_mpmc()
149 		if (head != ck_pr_load_ptr(&fifo->head))  in ck_hp_fifo_dequeue_mpmc()
152 		next = ck_pr_load_ptr(&head->next);  in ck_hp_fifo_dequeue_mpmc()
154 		if (head != ck_pr_load_ptr(&fifo->head))  in ck_hp_fifo_dequeue_mpmc()
157 		if (head == tail) {  in ck_hp_fifo_dequeue_mpmc()
161 			ck_pr_cas_ptr(&fifo->tail, tail, next);  in ck_hp_fifo_dequeue_mpmc()
163 		} else if (ck_pr_cas_ptr(&fifo->head, head, next) == true)  in ck_hp_fifo_dequeue_mpmc()
167 	ck_pr_store_ptr_unsafe(value, next->value);  in ck_hp_fifo_dequeue_mpmc()
168 	return head;  in ck_hp_fifo_dequeue_mpmc()
176 	struct ck_hp_fifo_entry *head, *tail, *next;  in ck_hp_fifo_trydequeue_mpmc()  local
178 	head = ck_pr_load_ptr(&fifo->head);  in ck_hp_fifo_trydequeue_mpmc()
180 	tail = ck_pr_load_ptr(&fifo->tail);  in ck_hp_fifo_trydequeue_mpmc()
181 	ck_hp_set_fence(record, 0, head);  in ck_hp_fifo_trydequeue_mpmc()
182 	if (head != ck_pr_load_ptr(&fifo->head))  in ck_hp_fifo_trydequeue_mpmc()
185 	next = ck_pr_load_ptr(&head->next);  in ck_hp_fifo_trydequeue_mpmc()
187 	if (head != ck_pr_load_ptr(&fifo->head))  in ck_hp_fifo_trydequeue_mpmc()
190 	if (head == tail) {  in ck_hp_fifo_trydequeue_mpmc()
194 		ck_pr_cas_ptr(&fifo->tail, tail, next);  in ck_hp_fifo_trydequeue_mpmc()
196 	} else if (ck_pr_cas_ptr(&fifo->head, head, next) == false)  in ck_hp_fifo_trydequeue_mpmc()
199 	ck_pr_store_ptr_unsafe(value, next->value);  in ck_hp_fifo_trydequeue_mpmc()
200 	return head;  in ck_hp_fifo_trydequeue_mpmc()
203 #define CK_HP_FIFO_ISEMPTY(f) ((f)->head->next == NULL)
204 #define CK_HP_FIFO_FIRST(f)   ((f)->head->next)
205 #define CK_HP_FIFO_NEXT(m)    ((m)->next)
210 #define CK_HP_FIFO_FOREACH_SAFE(fifo, entry, T)			\  argument
212              (entry) != NULL && ((T) = (entry)->next, 1);	\
213              (entry) = (T))