xref: /netbsd-src/external/bsd/openldap/dist/servers/lloadd/operation.c (revision 549b59ed3ccf0d36d3097190a0db27b770f3a839)
1 /*	$NetBSD: operation.c,v 1.2 2021/08/14 16:14:58 christos Exp $	*/
2 
3 /* $OpenLDAP$ */
4 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5  *
6  * Copyright 1998-2021 The OpenLDAP Foundation.
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted only as authorized by the OpenLDAP
11  * Public License.
12  *
13  * A copy of this license is available in the file LICENSE in the
14  * top-level directory of the distribution or, alternatively, at
15  * <http://www.OpenLDAP.org/license.html>.
16  */
17 
18 #include <sys/cdefs.h>
19 __RCSID("$NetBSD: operation.c,v 1.2 2021/08/14 16:14:58 christos Exp $");
20 
21 #include "portable.h"
22 
23 #include "lutil.h"
24 #include "lload.h"
25 
26 ldap_pvt_thread_mutex_t lload_pin_mutex;
27 unsigned long lload_next_pin = 1;
28 
29 ber_tag_t
slap_req2res(ber_tag_t tag)30 slap_req2res( ber_tag_t tag )
31 {
32     switch ( tag ) {
33         case LDAP_REQ_ADD:
34         case LDAP_REQ_BIND:
35         case LDAP_REQ_COMPARE:
36         case LDAP_REQ_EXTENDED:
37         case LDAP_REQ_MODIFY:
38         case LDAP_REQ_MODRDN:
39             tag++;
40             break;
41 
42         case LDAP_REQ_DELETE:
43             tag = LDAP_RES_DELETE;
44             break;
45 
46         case LDAP_REQ_ABANDON:
47         case LDAP_REQ_UNBIND:
48             tag = LBER_SEQUENCE;
49             break;
50 
51         case LDAP_REQ_SEARCH:
52             tag = LDAP_RES_SEARCH_RESULT;
53             break;
54 
55         default:
56             tag = LBER_SEQUENCE;
57     }
58 
59     return tag;
60 }
61 
62 const char *
lload_msgtype2str(ber_tag_t tag)63 lload_msgtype2str( ber_tag_t tag )
64 {
65     switch ( tag ) {
66         case LDAP_REQ_ABANDON: return "abandon request";
67         case LDAP_REQ_ADD: return "add request";
68         case LDAP_REQ_BIND: return "bind request";
69         case LDAP_REQ_COMPARE: return "compare request";
70         case LDAP_REQ_DELETE: return "delete request";
71         case LDAP_REQ_EXTENDED: return "extended request";
72         case LDAP_REQ_MODIFY: return "modify request";
73         case LDAP_REQ_RENAME: return "rename request";
74         case LDAP_REQ_SEARCH: return "search request";
75         case LDAP_REQ_UNBIND: return "unbind request";
76 
77         case LDAP_RES_ADD: return "add result";
78         case LDAP_RES_BIND: return "bind result";
79         case LDAP_RES_COMPARE: return "compare result";
80         case LDAP_RES_DELETE: return "delete result";
81         case LDAP_RES_EXTENDED: return "extended result";
82         case LDAP_RES_INTERMEDIATE: return "intermediate response";
83         case LDAP_RES_MODIFY: return "modify result";
84         case LDAP_RES_RENAME: return "rename result";
85         case LDAP_RES_SEARCH_ENTRY: return "search-entry response";
86         case LDAP_RES_SEARCH_REFERENCE: return "search-reference response";
87         case LDAP_RES_SEARCH_RESULT: return "search result";
88     }
89     return "unknown message";
90 }
91 
92 int
operation_client_cmp(const void * left,const void * right)93 operation_client_cmp( const void *left, const void *right )
94 {
95     const LloadOperation *l = left, *r = right;
96 
97     assert( l->o_client_connid == r->o_client_connid );
98     if ( l->o_client_msgid || r->o_client_msgid ) {
99         return ( l->o_client_msgid < r->o_client_msgid ) ?
100                 -1 :
101                 ( l->o_client_msgid > r->o_client_msgid );
102     } else {
103         return ( l->o_pin_id < r->o_pin_id ) ? -1 :
104                 ( l->o_pin_id > r->o_pin_id );
105     }
106 }
107 
108 int
operation_upstream_cmp(const void * left,const void * right)109 operation_upstream_cmp( const void *left, const void *right )
110 {
111     const LloadOperation *l = left, *r = right;
112 
113     assert( l->o_upstream_connid == r->o_upstream_connid );
114     if ( l->o_upstream_msgid || r->o_upstream_msgid ) {
115         return ( l->o_upstream_msgid < r->o_upstream_msgid ) ?
116                 -1 :
117                 ( l->o_upstream_msgid > r->o_upstream_msgid );
118     } else {
119         return ( l->o_pin_id < r->o_pin_id ) ? -1 :
120                 ( l->o_pin_id > r->o_pin_id );
121     }
122 }
123 
124 /*
125  * Entered holding c_mutex for now.
126  */
127 LloadOperation *
operation_init(LloadConnection * c,BerElement * ber)128 operation_init( LloadConnection *c, BerElement *ber )
129 {
130     LloadOperation *op;
131     ber_tag_t tag;
132     ber_len_t len;
133     int rc;
134 
135     if ( !IS_ALIVE( c, c_live ) ) {
136         return NULL;
137     }
138 
139     op = ch_calloc( 1, sizeof(LloadOperation) );
140     op->o_client = c;
141     op->o_client_connid = c->c_connid;
142     op->o_ber = ber;
143     op->o_start = slap_get_time();
144 
145     ldap_pvt_thread_mutex_init( &op->o_link_mutex );
146 
147     op->o_refcnt = 1;
148 
149     tag = ber_get_int( ber, &op->o_client_msgid );
150     if ( tag != LDAP_TAG_MSGID ) {
151         goto fail;
152     }
153 
154     if ( !op->o_client_msgid ) {
155         goto fail;
156     }
157 
158     CONNECTION_ASSERT_LOCKED(c);
159     rc = ldap_tavl_insert( &c->c_ops, op, operation_client_cmp, ldap_avl_dup_error );
160     if ( rc ) {
161         Debug( LDAP_DEBUG_PACKETS, "operation_init: "
162                 "several operations with same msgid=%d in-flight "
163                 "from client connid=%lu\n",
164                 op->o_client_msgid, op->o_client_connid );
165         goto fail;
166     }
167 
168     tag = op->o_tag = ber_skip_element( ber, &op->o_request );
169     switch ( tag ) {
170         case LBER_ERROR:
171             rc = -1;
172             break;
173     }
174     if ( rc ) {
175         ldap_tavl_delete( &c->c_ops, op, operation_client_cmp );
176         goto fail;
177     }
178 
179     tag = ber_peek_tag( ber, &len );
180     if ( tag == LDAP_TAG_CONTROLS ) {
181         ber_skip_element( ber, &op->o_ctrls );
182     }
183 
184     switch ( op->o_tag ) {
185         case LDAP_REQ_BIND:
186             lload_stats.counters[LLOAD_STATS_OPS_BIND].lc_ops_received++;
187             break;
188         default:
189             lload_stats.counters[LLOAD_STATS_OPS_OTHER].lc_ops_received++;
190             break;
191     }
192 
193     Debug( LDAP_DEBUG_STATS, "operation_init: "
194             "received a new operation, %s with msgid=%d for client "
195             "connid=%lu\n",
196             lload_msgtype2str( op->o_tag ), op->o_client_msgid,
197             op->o_client_connid );
198 
199     c->c_n_ops_executing++;
200     return op;
201 
202 fail:
203     ch_free( op );
204     return NULL;
205 }
206 
207 void
operation_destroy(LloadOperation * op)208 operation_destroy( LloadOperation *op )
209 {
210     Debug( LDAP_DEBUG_TRACE, "operation_destroy: "
211             "op=%p destroyed operation from client connid=%lu, "
212             "client msgid=%d\n",
213             op, op->o_client_connid, op->o_client_msgid );
214 
215     assert( op->o_refcnt == 0 );
216     assert( op->o_client == NULL );
217     assert( op->o_upstream == NULL );
218 
219     ber_free( op->o_ber, 1 );
220     ldap_pvt_thread_mutex_destroy( &op->o_link_mutex );
221     ch_free( op );
222 }
223 
224 int
operation_unlink(LloadOperation * op)225 operation_unlink( LloadOperation *op )
226 {
227     LloadConnection *client, *upstream;
228     uintptr_t prev_refcnt;
229     int result = 0;
230 
231     if ( !( prev_refcnt = try_release_ref(
232                     &op->o_refcnt, op, (dispose_cb *)operation_destroy ) ) ) {
233         return result;
234     }
235 
236     assert( prev_refcnt == 1 );
237 
238     Debug( LDAP_DEBUG_TRACE, "operation_unlink: "
239             "unlinking operation between client connid=%lu and upstream "
240             "connid=%lu "
241             "client msgid=%d\n",
242             op->o_client_connid, op->o_upstream_connid, op->o_client_msgid );
243 
244     checked_lock( &op->o_link_mutex );
245     client = op->o_client;
246     upstream = op->o_upstream;
247 
248     op->o_client = NULL;
249     op->o_upstream = NULL;
250     checked_unlock( &op->o_link_mutex );
251 
252     assert( client || upstream );
253 
254     if ( client ) {
255         result |= operation_unlink_client( op, client );
256         operation_update_global_rejected( op );
257     }
258 
259     if ( upstream ) {
260         result |= operation_unlink_upstream( op, upstream );
261     }
262 
263     return result;
264 }
265 
266 int
operation_unlink_client(LloadOperation * op,LloadConnection * client)267 operation_unlink_client( LloadOperation *op, LloadConnection *client )
268 {
269     LloadOperation *removed;
270     int result = 0;
271 
272     Debug( LDAP_DEBUG_TRACE, "operation_unlink_client: "
273             "unlinking operation op=%p msgid=%d client connid=%lu\n",
274             op, op->o_client_msgid, op->o_client_connid );
275 
276     CONNECTION_LOCK(client);
277     if ( (removed = ldap_tavl_delete(
278                    &client->c_ops, op, operation_client_cmp )) ) {
279         result = LLOAD_OP_DETACHING_CLIENT;
280 
281         assert( op == removed );
282         client->c_n_ops_executing--;
283 
284         if ( client->c_state == LLOAD_C_BINDING ) {
285             client->c_state = LLOAD_C_READY;
286             if ( !BER_BVISNULL( &client->c_auth ) ) {
287                 ber_memfree( client->c_auth.bv_val );
288                 BER_BVZERO( &client->c_auth );
289             }
290             if ( !BER_BVISNULL( &client->c_sasl_bind_mech ) ) {
291                 ber_memfree( client->c_sasl_bind_mech.bv_val );
292                 BER_BVZERO( &client->c_sasl_bind_mech );
293             }
294             if ( op->o_pin_id ) {
295                 client->c_pin_id = 0;
296             }
297         }
298     }
299     if ( client->c_state == LLOAD_C_CLOSING && !client->c_ops ) {
300         CONNECTION_DESTROY(client);
301     } else {
302         CONNECTION_UNLOCK(client);
303     }
304 
305     return result;
306 }
307 
308 int
operation_unlink_upstream(LloadOperation * op,LloadConnection * upstream)309 operation_unlink_upstream( LloadOperation *op, LloadConnection *upstream )
310 {
311     LloadOperation *removed;
312     LloadBackend *b = NULL;
313     int result = 0;
314 
315     Debug( LDAP_DEBUG_TRACE, "operation_unlink_upstream: "
316             "unlinking operation op=%p msgid=%d upstream connid=%lu\n",
317             op, op->o_upstream_msgid, op->o_upstream_connid );
318 
319     CONNECTION_LOCK(upstream);
320     if ( (removed = ldap_tavl_delete(
321                    &upstream->c_ops, op, operation_upstream_cmp )) ) {
322         result |= LLOAD_OP_DETACHING_UPSTREAM;
323 
324         assert( op == removed );
325         upstream->c_n_ops_executing--;
326 
327         if ( upstream->c_state == LLOAD_C_BINDING ) {
328             assert( op->o_tag == LDAP_REQ_BIND && upstream->c_ops == NULL );
329             upstream->c_state = LLOAD_C_READY;
330             if ( !BER_BVISNULL( &upstream->c_sasl_bind_mech ) ) {
331                 ber_memfree( upstream->c_sasl_bind_mech.bv_val );
332                 BER_BVZERO( &upstream->c_sasl_bind_mech );
333             }
334         }
335         operation_update_conn_counters( op, upstream );
336         b = upstream->c_backend;
337     }
338     if ( upstream->c_state == LLOAD_C_CLOSING && !upstream->c_ops ) {
339         CONNECTION_DESTROY(upstream);
340     } else {
341         CONNECTION_UNLOCK(upstream);
342     }
343 
344     if ( b ) {
345         checked_lock( &b->b_mutex );
346         b->b_n_ops_executing--;
347         operation_update_backend_counters( op, b );
348         checked_unlock( &b->b_mutex );
349     }
350 
351     return result;
352 }
353 
354 int
operation_send_abandon(LloadOperation * op,LloadConnection * upstream)355 operation_send_abandon( LloadOperation *op, LloadConnection *upstream )
356 {
357     BerElement *ber;
358     int rc = -1;
359 
360     if ( !IS_ALIVE( upstream, c_live ) ) {
361         return rc;
362     }
363 
364     checked_lock( &upstream->c_io_mutex );
365     ber = upstream->c_pendingber;
366     if ( ber == NULL && (ber = ber_alloc()) == NULL ) {
367         Debug( LDAP_DEBUG_ANY, "operation_send_abandon: "
368                 "ber_alloc failed\n" );
369         goto done;
370     }
371     upstream->c_pendingber = ber;
372 
373     Debug( LDAP_DEBUG_TRACE, "operation_send_abandon: "
374             "abandoning %s msgid=%d on connid=%lu\n",
375             lload_msgtype2str( op->o_tag ), op->o_upstream_msgid,
376             op->o_upstream_connid );
377 
378     if ( op->o_tag == LDAP_REQ_BIND ) {
379         rc = ber_printf( ber, "t{tit{ist{s}}}", LDAP_TAG_MESSAGE,
380                 LDAP_TAG_MSGID, upstream->c_next_msgid++,
381                 LDAP_REQ_BIND, LDAP_VERSION3, "", LDAP_AUTH_SASL, "" );
382     } else {
383         rc = ber_printf( ber, "t{titi}", LDAP_TAG_MESSAGE,
384                 LDAP_TAG_MSGID, upstream->c_next_msgid++,
385                 LDAP_REQ_ABANDON, op->o_upstream_msgid );
386     }
387 
388     if ( rc < 0 ) {
389         ber_free( ber, 1 );
390         upstream->c_pendingber = NULL;
391         goto done;
392     }
393     rc = LDAP_SUCCESS;
394 
395 done:
396     checked_unlock( &upstream->c_io_mutex );
397     return rc;
398 }
399 
400 /*
401  * Will remove the operation from its upstream and if it was still there,
402  * sends an abandon request.
403  *
404  * Being called from client_reset or request_abandon, the following hold:
405  * - noone else is processing the read part of the client connection (no new
406  *   operations come in there - relevant for the c_state checks)
407  * - op->o_client_refcnt > op->o_client_live (and it follows that op->o_client != NULL)
408  */
409 void
operation_abandon(LloadOperation * op)410 operation_abandon( LloadOperation *op )
411 {
412     LloadConnection *c;
413 
414     checked_lock( &op->o_link_mutex );
415     c = op->o_upstream;
416     checked_unlock( &op->o_link_mutex );
417     if ( !c || !IS_ALIVE( c, c_live ) ) {
418         goto done;
419     }
420 
421     /* for now consider all abandoned operations completed,
422      * perhaps add a separate counter later */
423     op->o_res = LLOAD_OP_COMPLETED;
424     if ( !operation_unlink_upstream( op, c ) ) {
425         /* The operation has already been abandoned or finished */
426         Debug( LDAP_DEBUG_TRACE, "operation_abandon: "
427                 "%s from connid=%lu msgid=%d not present in connid=%lu any "
428                 "more\n",
429                 lload_msgtype2str( op->o_tag ), op->o_client_connid,
430                 op->o_client_msgid, op->o_upstream_connid );
431         goto done;
432     }
433 
434     if ( operation_send_abandon( op, c ) == LDAP_SUCCESS ) {
435         connection_write_cb( -1, 0, c );
436     }
437 
438 done:
439     operation_unlink( op );
440 }
441 
442 void
operation_send_reject(LloadOperation * op,int result,const char * msg,int send_anyway)443 operation_send_reject(
444         LloadOperation *op,
445         int result,
446         const char *msg,
447         int send_anyway )
448 {
449     LloadConnection *c;
450     BerElement *ber;
451     int found;
452 
453     Debug( LDAP_DEBUG_TRACE, "operation_send_reject: "
454             "rejecting %s from client connid=%lu with message: \"%s\"\n",
455             lload_msgtype2str( op->o_tag ), op->o_client_connid, msg );
456 
457     checked_lock( &op->o_link_mutex );
458     c = op->o_client;
459     checked_unlock( &op->o_link_mutex );
460     if ( !c || !IS_ALIVE( c, c_live ) ) {
461         Debug( LDAP_DEBUG_TRACE, "operation_send_reject: "
462                 "not sending msgid=%d, client connid=%lu is dead\n",
463                 op->o_client_msgid, op->o_client_connid );
464 
465         goto done;
466     }
467 
468     found = operation_unlink_client( op, c );
469     if ( !found && !send_anyway ) {
470         Debug( LDAP_DEBUG_TRACE, "operation_send_reject: "
471                 "msgid=%d not scheduled for client connid=%lu anymore, "
472                 "not sending\n",
473                 op->o_client_msgid, c->c_connid );
474         goto done;
475     }
476 
477     if ( op->o_client_msgid == 0 ) {
478         assert( op->o_saved_msgid == 0 && op->o_pin_id );
479         Debug( LDAP_DEBUG_TRACE, "operation_send_reject: "
480                 "operation pin=%lu is just a pin, not sending\n",
481                 op->o_pin_id );
482         goto done;
483     }
484 
485     checked_lock( &c->c_io_mutex );
486     ber = c->c_pendingber;
487     if ( ber == NULL && (ber = ber_alloc()) == NULL ) {
488         checked_unlock( &c->c_io_mutex );
489         Debug( LDAP_DEBUG_ANY, "operation_send_reject: "
490                 "ber_alloc failed, closing connid=%lu\n",
491                 c->c_connid );
492         CONNECTION_LOCK_DESTROY(c);
493         goto done;
494     }
495     c->c_pendingber = ber;
496 
497     ber_printf( ber, "t{tit{ess}}", LDAP_TAG_MESSAGE,
498             LDAP_TAG_MSGID, op->o_client_msgid,
499             slap_req2res( op->o_tag ), result, "", msg );
500 
501     checked_unlock( &c->c_io_mutex );
502 
503     connection_write_cb( -1, 0, c );
504 
505 done:
506     operation_unlink( op );
507 }
508 
509 /*
510  * Upstream is shutting down, signal the client if necessary, but we have to
511  * call operation_destroy_from_upstream ourselves to detach upstream from the
512  * op.
513  *
514  * Only called from upstream_destroy.
515  */
516 void
operation_lost_upstream(LloadOperation * op)517 operation_lost_upstream( LloadOperation *op )
518 {
519     operation_send_reject( op, LDAP_OTHER,
520             "connection to the remote server has been severed", 0 );
521 }
522 
523 int
connection_timeout(LloadConnection * upstream,void * arg)524 connection_timeout( LloadConnection *upstream, void *arg )
525 {
526     LloadOperation *op;
527     TAvlnode *ops = NULL, *node, *next;
528     LloadBackend *b = upstream->c_backend;
529     time_t threshold = *(time_t *)arg;
530     int rc, nops = 0;
531 
532     CONNECTION_LOCK(upstream);
533     for ( node = ldap_tavl_end( upstream->c_ops, TAVL_DIR_LEFT ); node &&
534             ((LloadOperation *)node->avl_data)->o_start <
535                     threshold; /* shortcut */
536             node = next ) {
537         LloadOperation *found_op;
538 
539         next = ldap_tavl_next( node, TAVL_DIR_RIGHT );
540         op = node->avl_data;
541 
542         /* Have we received another response since? */
543         if ( op->o_last_response && op->o_last_response >= threshold ) {
544             continue;
545         }
546 
547         op->o_res = LLOAD_OP_FAILED;
548         found_op = ldap_tavl_delete( &upstream->c_ops, op, operation_upstream_cmp );
549         assert( op == found_op );
550 
551         if ( upstream->c_state == LLOAD_C_BINDING ) {
552             assert( op->o_tag == LDAP_REQ_BIND && upstream->c_ops == NULL );
553             upstream->c_state = LLOAD_C_READY;
554             if ( !BER_BVISNULL( &upstream->c_sasl_bind_mech ) ) {
555                 ber_memfree( upstream->c_sasl_bind_mech.bv_val );
556                 BER_BVZERO( &upstream->c_sasl_bind_mech );
557             }
558         }
559 
560         rc = ldap_tavl_insert( &ops, op, operation_upstream_cmp, ldap_avl_dup_error );
561         assert( rc == LDAP_SUCCESS );
562 
563         Debug( LDAP_DEBUG_STATS2, "connection_timeout: "
564                 "timing out %s from connid=%lu msgid=%d sent to connid=%lu as "
565                 "msgid=%d\n",
566                 lload_msgtype2str( op->o_tag ), op->o_client_connid,
567                 op->o_client_msgid, op->o_upstream_connid,
568                 op->o_upstream_msgid );
569         nops++;
570     }
571 
572     if ( nops == 0 ) {
573         CONNECTION_UNLOCK(upstream);
574         return LDAP_SUCCESS;
575     }
576     upstream->c_n_ops_executing -= nops;
577     upstream->c_counters.lc_ops_failed += nops;
578     Debug( LDAP_DEBUG_STATS, "connection_timeout: "
579             "timing out %d operations for connid=%lu\n",
580             nops, upstream->c_connid );
581     CONNECTION_UNLOCK(upstream);
582 
583     checked_lock( &b->b_mutex );
584     b->b_n_ops_executing -= nops;
585     checked_unlock( &b->b_mutex );
586 
587     for ( node = ldap_tavl_end( ops, TAVL_DIR_LEFT ); node;
588             node = ldap_tavl_next( node, TAVL_DIR_RIGHT ) ) {
589         op = node->avl_data;
590 
591         operation_send_reject( op,
592                 op->o_tag == LDAP_REQ_SEARCH ? LDAP_TIMELIMIT_EXCEEDED :
593                                                LDAP_ADMINLIMIT_EXCEEDED,
594                 "upstream did not respond in time", 0 );
595 
596         if ( rc == LDAP_SUCCESS ) {
597             rc = operation_send_abandon( op, upstream );
598         }
599         operation_unlink( op );
600     }
601 
602     /* TODO: if operation_send_abandon failed, we need to kill the upstream */
603     if ( rc == LDAP_SUCCESS ) {
604         connection_write_cb( -1, 0, upstream );
605     }
606 
607     CONNECTION_LOCK(upstream);
608     if ( upstream->c_state == LLOAD_C_CLOSING && !upstream->c_ops ) {
609         CONNECTION_DESTROY(upstream);
610     } else {
611         CONNECTION_UNLOCK(upstream);
612     }
613 
614     /* just dispose of the AVL, most operations should already be gone */
615     ldap_tavl_free( ops, NULL );
616     return LDAP_SUCCESS;
617 }
618 
619 void
operations_timeout(evutil_socket_t s,short what,void * arg)620 operations_timeout( evutil_socket_t s, short what, void *arg )
621 {
622     struct event *self = arg;
623     LloadBackend *b;
624     time_t threshold;
625 
626     Debug( LDAP_DEBUG_TRACE, "operations_timeout: "
627             "running timeout task\n" );
628     if ( !lload_timeout_api ) goto done;
629 
630     threshold = slap_get_time() - lload_timeout_api->tv_sec;
631 
632     LDAP_CIRCLEQ_FOREACH ( b, &backend, b_next ) {
633         epoch_t epoch;
634 
635         checked_lock( &b->b_mutex );
636         if ( b->b_n_ops_executing == 0 ) {
637             checked_unlock( &b->b_mutex );
638             continue;
639         }
640 
641         epoch = epoch_join();
642 
643         Debug( LDAP_DEBUG_TRACE, "operations_timeout: "
644                 "timing out binds for backend uri=%s\n",
645                 b->b_uri.bv_val );
646         connections_walk_last( &b->b_mutex, &b->b_bindconns, b->b_last_bindconn,
647                 connection_timeout, &threshold );
648 
649         Debug( LDAP_DEBUG_TRACE, "operations_timeout: "
650                 "timing out other operations for backend uri=%s\n",
651                 b->b_uri.bv_val );
652         connections_walk_last( &b->b_mutex, &b->b_conns, b->b_last_conn,
653                 connection_timeout, &threshold );
654 
655         epoch_leave( epoch );
656         checked_unlock( &b->b_mutex );
657     }
658 done:
659     Debug( LDAP_DEBUG_TRACE, "operations_timeout: "
660             "timeout task finished\n" );
661     evtimer_add( self, lload_timeout_api );
662 }
663 
664 void
operation_update_global_rejected(LloadOperation * op)665 operation_update_global_rejected( LloadOperation *op )
666 {
667     if ( op->o_res == LLOAD_OP_REJECTED ) {
668         assert( op->o_upstream_connid == 0 );
669         switch ( op->o_tag ) {
670             case LDAP_REQ_BIND:
671                 lload_stats.counters[LLOAD_STATS_OPS_BIND].lc_ops_rejected++;
672                 break;
673             default:
674                 lload_stats.counters[LLOAD_STATS_OPS_OTHER].lc_ops_rejected++;
675                 break;
676         }
677     }
678 }
679 
680 void
operation_update_conn_counters(LloadOperation * op,LloadConnection * upstream)681 operation_update_conn_counters( LloadOperation *op, LloadConnection *upstream )
682 {
683     if ( op->o_res == LLOAD_OP_COMPLETED ) {
684         upstream->c_counters.lc_ops_completed++;
685     } else {
686         upstream->c_counters.lc_ops_failed++;
687     }
688 }
689 
690 void
operation_update_backend_counters(LloadOperation * op,LloadBackend * b)691 operation_update_backend_counters( LloadOperation *op, LloadBackend *b )
692 {
693     int stat_type = op->o_tag == LDAP_REQ_BIND ? LLOAD_STATS_OPS_BIND :
694                                                  LLOAD_STATS_OPS_OTHER;
695 
696     assert( b != NULL );
697     if ( op->o_res == LLOAD_OP_COMPLETED ) {
698         b->b_counters[stat_type].lc_ops_completed++;
699     } else {
700         b->b_counters[stat_type].lc_ops_failed++;
701     }
702 }
703