1 /* $NetBSD: tools.c,v 1.3 2021/08/14 16:15:00 christos Exp $ */
2
3 /* tools.c - tools for slap tools */
4 /* $OpenLDAP$ */
5 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
6 *
7 * Copyright 2011-2021 The OpenLDAP Foundation.
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted only as authorized by the OpenLDAP
12 * Public License.
13 *
14 * A copy of this license is available in the file LICENSE in the
15 * top-level directory of the distribution or, alternatively, at
16 * <http://www.OpenLDAP.org/license.html>.
17 */
18
19 #include <sys/cdefs.h>
20 __RCSID("$NetBSD: tools.c,v 1.3 2021/08/14 16:15:00 christos Exp $");
21
22 #include "portable.h"
23
24 #include <stdio.h>
25 #include <ac/string.h>
26 #include <ac/errno.h>
27
28 #define AVL_INTERNAL
29 #include "back-mdb.h"
30 #include "idl.h"
31
32 #ifdef MDB_TOOL_IDL_CACHING
33 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
34
35 #define IDBLOCK 1024
36
37 typedef struct mdb_tool_idl_cache_entry {
38 struct mdb_tool_idl_cache_entry *next;
39 ID ids[IDBLOCK];
40 } mdb_tool_idl_cache_entry;
41
42 typedef struct mdb_tool_idl_cache {
43 struct berval kstr;
44 mdb_tool_idl_cache_entry *head, *tail;
45 ID first, last;
46 int count;
47 short offset;
48 short flags;
49 } mdb_tool_idl_cache;
50 #define WAS_FOUND 0x01
51 #define WAS_RANGE 0x02
52
53 #define MDB_TOOL_IDL_FLUSH(be, txn) mdb_tool_idl_flush(be, txn)
54 #else
55 #define MDB_TOOL_IDL_FLUSH(be, txn)
56 #endif /* MDB_TOOL_IDL_CACHING */
57
58 MDB_txn *mdb_tool_txn = NULL;
59
60 static MDB_txn *txi = NULL;
61 static MDB_cursor *cursor = NULL, *idcursor = NULL;
62 static MDB_cursor *mcp = NULL, *mcd = NULL;
63 static MDB_val key, data;
64 static ID previd = NOID;
65
66 typedef struct dn_id {
67 ID id;
68 struct berval dn;
69 } dn_id;
70
71 #define HOLE_SIZE 4096
72 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
73 static unsigned nhmax = HOLE_SIZE;
74 static unsigned nholes;
75
76 static struct berval *tool_base;
77 static int tool_scope;
78 static Filter *tool_filter;
79 static Entry *tool_next_entry;
80
81 static ID mdb_tool_ix_id;
82 static BackendDB *mdb_tool_ix_be;
83 static MDB_txn *mdb_tool_ix_txn;
84 static int mdb_tool_index_tcount, mdb_tool_threads;
85 static IndexRec *mdb_tool_index_rec;
86 static AttrIxInfo **mdb_tool_axinfo;
87 static struct mdb_info *mdb_tool_info;
88 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
89 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
90 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
91 static void * mdb_tool_index_task( void *ctx, void *ptr );
92
93 static int mdb_writes, mdb_writes_per_commit;
94
95 /* Number of ops per commit in Quick mode.
96 * Batching speeds writes overall, but too large a
97 * batch will fail with MDB_TXN_FULL.
98 */
99 #ifndef MDB_WRITES_PER_COMMIT
100 #define MDB_WRITES_PER_COMMIT 500
101 #endif
102
103 static int
104 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
105
mdb_tool_entry_open(BackendDB * be,int mode)106 int mdb_tool_entry_open(
107 BackendDB *be, int mode )
108 {
109 /* In Quick mode, commit once per 500 entries */
110 mdb_writes = 0;
111 if ( slapMode & SLAP_TOOL_QUICK )
112 mdb_writes_per_commit = MDB_WRITES_PER_COMMIT;
113 else
114 mdb_writes_per_commit = 1;
115
116 #ifdef MDB_TOOL_IDL_CACHING /* threaded indexing has no performance advantage */
117 /* Set up for threaded slapindex */
118 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
119 if ( !mdb_tool_info ) {
120 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
121 ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
122 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
123 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
124 if ( mdb->mi_nattrs ) {
125 int i;
126 mdb_tool_threads = slap_tool_thread_max - 1;
127 if ( mdb_tool_threads > 1 ) {
128 mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec ));
129 mdb_tool_axinfo = ch_calloc( mdb_tool_threads, sizeof( AttrIxInfo* ) +
130 sizeof( AttrIxInfo ));
131 mdb_tool_axinfo[0] = (AttrIxInfo *)(mdb_tool_axinfo + mdb_tool_threads);
132 for (i=1; i<mdb_tool_threads; i++)
133 mdb_tool_axinfo[i] = mdb_tool_axinfo[i-1]+1;
134 mdb_tool_index_tcount = mdb_tool_threads - 1;
135 mdb_tool_ix_be = be;
136 for (i=1; i<mdb_tool_threads; i++) {
137 int *ptr = ch_malloc( sizeof( int ));
138 *ptr = i;
139 ldap_pvt_thread_pool_submit( &connection_pool,
140 mdb_tool_index_task, ptr );
141 }
142 mdb_tool_info = mdb;
143 }
144 }
145 }
146 }
147 #endif
148
149 return 0;
150 }
151
mdb_tool_entry_close(BackendDB * be)152 int mdb_tool_entry_close(
153 BackendDB *be )
154 {
155 #ifdef MDB_TOOL_IDL_CACHING
156 if ( mdb_tool_info ) {
157 int i;
158 slapd_shutdown = 1;
159 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
160
161 /* There might still be some threads starting */
162 while ( mdb_tool_index_tcount > 0 ) {
163 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
164 &mdb_tool_index_mutex );
165 }
166
167 mdb_tool_index_tcount = mdb_tool_threads - 1;
168 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
169
170 /* Make sure all threads are stopped */
171 while ( mdb_tool_index_tcount > 0 ) {
172 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
173 &mdb_tool_index_mutex );
174 }
175 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
176
177 mdb_tool_info = NULL;
178 slapd_shutdown = 0;
179 ch_free( mdb_tool_index_rec );
180 mdb_tool_index_tcount = mdb_tool_threads - 1;
181 if (mdb_tool_txn)
182 MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
183 for (i=0; i<mdb_tool_threads; i++) {
184 mdb_tool_idl_cache *ic;
185 mdb_tool_idl_cache_entry *ice;
186 while ((ic = mdb_tool_axinfo[i]->ai_clist)) {
187 mdb_tool_axinfo[i]->ai_clist = ic->head;
188 free(ic);
189 }
190 while ((ice = mdb_tool_axinfo[i]->ai_flist)) {
191 mdb_tool_axinfo[i]->ai_flist = ice->next;
192 free(ice);
193 }
194 }
195 }
196 #endif
197
198 if( idcursor ) {
199 mdb_cursor_close( idcursor );
200 idcursor = NULL;
201 }
202 if( cursor ) {
203 mdb_cursor_close( cursor );
204 cursor = NULL;
205 }
206 {
207 struct mdb_info *mdb = be->be_private;
208 if ( mdb ) {
209 int i;
210 for (i=0; i<mdb->mi_nattrs; i++)
211 mdb->mi_attrs[i]->ai_cursor = NULL;
212 }
213 }
214 if( mdb_tool_txn ) {
215 int rc;
216 if (( rc = mdb_txn_commit( mdb_tool_txn ))) {
217 Debug( LDAP_DEBUG_ANY,
218 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: "
219 "txn_commit failed: %s (%d)\n",
220 be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
221 return -1;
222 }
223 mdb_tool_txn = NULL;
224 }
225 if( txi ) {
226 int rc;
227 if (( rc = mdb_txn_commit( txi ))) {
228 Debug( LDAP_DEBUG_ANY,
229 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: "
230 "txn_commit failed: %s (%d)\n",
231 be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
232 return -1;
233 }
234 txi = NULL;
235 }
236
237 if( nholes ) {
238 unsigned i;
239 fprintf( stderr, "Error, entries missing!\n");
240 for (i=0; i<nholes; i++) {
241 fprintf(stderr, " entry %ld: %s\n",
242 holes[i].id, holes[i].dn.bv_val);
243 }
244 nholes = 0;
245 return -1;
246 }
247
248 return 0;
249 }
250
251 ID
mdb_tool_entry_first_x(BackendDB * be,struct berval * base,int scope,Filter * f)252 mdb_tool_entry_first_x(
253 BackendDB *be,
254 struct berval *base,
255 int scope,
256 Filter *f )
257 {
258 tool_base = base;
259 tool_scope = scope;
260 tool_filter = f;
261
262 return mdb_tool_entry_next( be );
263 }
264
mdb_tool_entry_next(BackendDB * be)265 ID mdb_tool_entry_next(
266 BackendDB *be )
267 {
268 int rc;
269 ID id;
270 struct mdb_info *mdb;
271
272 assert( be != NULL );
273 assert( slapMode & SLAP_TOOL_MODE );
274
275 mdb = (struct mdb_info *) be->be_private;
276 assert( mdb != NULL );
277
278 if ( !mdb_tool_txn ) {
279 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
280 if ( rc )
281 return NOID;
282 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
283 if ( rc ) {
284 mdb_txn_abort( mdb_tool_txn );
285 return NOID;
286 }
287 }
288
289 next:;
290 rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
291
292 if( rc ) {
293 return NOID;
294 }
295
296 previd = *(ID *)key.mv_data;
297 id = previd;
298
299 if ( !data.mv_size )
300 goto next;
301
302 if ( tool_filter || tool_base ) {
303 static Operation op = {0};
304 static Opheader ohdr = {0};
305
306 op.o_hdr = &ohdr;
307 op.o_bd = be;
308 op.o_tmpmemctx = NULL;
309 op.o_tmpmfuncs = &ch_mfuncs;
310
311 if ( tool_next_entry ) {
312 mdb_entry_release( &op, tool_next_entry, 0 );
313 tool_next_entry = NULL;
314 }
315
316 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
317 if ( rc == LDAP_NO_SUCH_OBJECT ) {
318 goto next;
319 }
320
321 assert( tool_next_entry != NULL );
322
323 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
324 {
325 mdb_entry_release( &op, tool_next_entry, 0 );
326 tool_next_entry = NULL;
327 goto next;
328 }
329 }
330
331 return id;
332 }
333
mdb_tool_dn2id_get(Backend * be,struct berval * dn)334 ID mdb_tool_dn2id_get(
335 Backend *be,
336 struct berval *dn
337 )
338 {
339 struct mdb_info *mdb;
340 Operation op = {0};
341 Opheader ohdr = {0};
342 ID id;
343 int rc;
344
345 if ( BER_BVISEMPTY(dn) )
346 return 0;
347
348 mdb = (struct mdb_info *) be->be_private;
349
350 if ( !mdb_tool_txn ) {
351 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
352 MDB_RDONLY : 0, &mdb_tool_txn );
353 if ( rc )
354 return NOID;
355 }
356
357 op.o_hdr = &ohdr;
358 op.o_bd = be;
359 op.o_tmpmemctx = NULL;
360 op.o_tmpmfuncs = &ch_mfuncs;
361
362 rc = mdb_dn2id( &op, mdb_tool_txn, NULL, dn, &id, NULL, NULL, NULL );
363 if ( rc == MDB_NOTFOUND )
364 return NOID;
365
366 return id;
367 }
368
369 static int
mdb_tool_entry_get_int(BackendDB * be,ID id,Entry ** ep)370 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
371 {
372 Operation op = {0};
373 Opheader ohdr = {0};
374
375 Entry *e = NULL;
376 struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
377 int rc;
378
379 assert( be != NULL );
380 assert( slapMode & SLAP_TOOL_MODE );
381
382 if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
383 *ep = tool_next_entry;
384 tool_next_entry = NULL;
385 return LDAP_SUCCESS;
386 }
387
388 if ( id != previd ) {
389 key.mv_size = sizeof(ID);
390 key.mv_data = &id;
391 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
392 if ( rc ) {
393 rc = LDAP_OTHER;
394 goto done;
395 }
396 }
397 if ( !data.mv_size ) {
398 rc = LDAP_NO_SUCH_OBJECT;
399 goto done;
400 }
401
402 op.o_hdr = &ohdr;
403 op.o_bd = be;
404 op.o_tmpmemctx = NULL;
405 op.o_tmpmfuncs = &ch_mfuncs;
406 if ( slapMode & SLAP_TOOL_READONLY ) {
407 rc = mdb_id2name( &op, mdb_tool_txn, &idcursor, id, &dn, &ndn );
408 if ( rc ) {
409 rc = LDAP_OTHER;
410 goto done;
411 }
412 if ( tool_base != NULL ) {
413 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
414 ch_free( dn.bv_val );
415 ch_free( ndn.bv_val );
416 rc = LDAP_NO_SUCH_OBJECT;
417 goto done;
418 }
419 }
420 }
421 rc = mdb_entry_decode( &op, mdb_tool_txn, &data, id, &e );
422 e->e_id = id;
423 if ( !BER_BVISNULL( &dn )) {
424 e->e_name = dn;
425 e->e_nname = ndn;
426 } else {
427 e->e_name.bv_val = NULL;
428 e->e_nname.bv_val = NULL;
429 }
430
431 done:
432 if ( e != NULL ) {
433 *ep = e;
434 }
435
436 return rc;
437 }
438
439 Entry*
mdb_tool_entry_get(BackendDB * be,ID id)440 mdb_tool_entry_get( BackendDB *be, ID id )
441 {
442 Entry *e = NULL;
443 int rc;
444
445 if ( !mdb_tool_txn ) {
446 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
447 rc = mdb_txn_begin( mdb->mi_dbenv, NULL,
448 (slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &mdb_tool_txn );
449 if ( rc )
450 return NULL;
451 }
452 if ( !cursor ) {
453 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
454 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
455 if ( rc ) {
456 mdb_txn_abort( mdb_tool_txn );
457 mdb_tool_txn = NULL;
458 return NULL;
459 }
460 }
461 (void)mdb_tool_entry_get_int( be, id, &e );
462 return e;
463 }
464
mdb_tool_next_id(Operation * op,MDB_txn * tid,Entry * e,struct berval * text,int hole)465 static int mdb_tool_next_id(
466 Operation *op,
467 MDB_txn *tid,
468 Entry *e,
469 struct berval *text,
470 int hole )
471 {
472 struct berval dn = e->e_name;
473 struct berval ndn = e->e_nname;
474 struct berval pdn, npdn, nmatched;
475 ID id, pid = 0;
476 int rc;
477
478 if (ndn.bv_len == 0) {
479 e->e_id = 0;
480 return 0;
481 }
482
483 rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, NULL, &nmatched );
484 if ( rc == MDB_NOTFOUND ) {
485 if ( !be_issuffix( op->o_bd, &ndn ) ) {
486 ID eid = e->e_id;
487 dnParent( &ndn, &npdn );
488 if ( nmatched.bv_len != npdn.bv_len ) {
489 dnParent( &dn, &pdn );
490 e->e_name = pdn;
491 e->e_nname = npdn;
492 rc = mdb_tool_next_id( op, tid, e, text, 1 );
493 e->e_name = dn;
494 e->e_nname = ndn;
495 if ( rc ) {
496 return rc;
497 }
498 /* If parent didn't exist, it was created just now
499 * and its ID is now in e->e_id. Make sure the current
500 * entry gets added under the new parent ID.
501 */
502 if ( eid != e->e_id ) {
503 pid = e->e_id;
504 }
505 } else {
506 pid = id;
507 }
508 }
509 rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
510 if ( rc ) {
511 snprintf( text->bv_val, text->bv_len,
512 "next_id failed: %s (%d)",
513 mdb_strerror(rc), rc );
514 Debug( LDAP_DEBUG_ANY,
515 "=> mdb_tool_next_id: %s\n", text->bv_val );
516 return rc;
517 }
518 rc = mdb_dn2id_add( op, mcp, mcd, pid, 1, 1, e );
519 if ( rc ) {
520 snprintf( text->bv_val, text->bv_len,
521 "dn2id_add failed: %s (%d)",
522 mdb_strerror(rc), rc );
523 Debug( LDAP_DEBUG_ANY,
524 "=> mdb_tool_next_id: %s\n", text->bv_val );
525 } else if ( hole ) {
526 MDB_val key, data;
527 if ( nholes == nhmax - 1 ) {
528 if ( holes == hbuf ) {
529 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
530 AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
531 } else {
532 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
533 }
534 nhmax *= 2;
535 }
536 ber_dupbv( &holes[nholes].dn, &ndn );
537 holes[nholes++].id = e->e_id;
538 key.mv_size = sizeof(ID);
539 key.mv_data = &e->e_id;
540 data.mv_size = 0;
541 data.mv_data = NULL;
542 rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
543 if ( rc == MDB_KEYEXIST )
544 rc = 0;
545 if ( rc ) {
546 snprintf( text->bv_val, text->bv_len,
547 "dummy id2entry add failed: %s (%d)",
548 mdb_strerror(rc), rc );
549 Debug( LDAP_DEBUG_ANY,
550 "=> mdb_tool_next_id: %s\n", text->bv_val );
551 }
552 }
553 } else if ( !hole ) {
554 unsigned i, j;
555
556 e->e_id = id;
557
558 for ( i=0; i<nholes; i++) {
559 if ( holes[i].id == e->e_id ) {
560 free(holes[i].dn.bv_val);
561 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
562 holes[j].id = 0;
563 nholes--;
564 break;
565 } else if ( holes[i].id > e->e_id ) {
566 break;
567 }
568 }
569 }
570 return rc;
571 }
572
573 static int
mdb_tool_index_add(Operation * op,MDB_txn * txn,Entry * e)574 mdb_tool_index_add(
575 Operation *op,
576 MDB_txn *txn,
577 Entry *e )
578 {
579 struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
580
581 if ( !mdb->mi_nattrs )
582 return 0;
583
584 if ( mdb_tool_threads > 1 ) {
585 IndexRec *ir;
586 int i, rc;
587 Attribute *a;
588
589 ir = mdb_tool_index_rec;
590 for (i=0; i<mdb->mi_nattrs; i++)
591 ir[i].ir_attrs = NULL;
592
593 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
594 rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
595 &a->a_desc->ad_tags, ir );
596 if ( rc )
597 return rc;
598 }
599 for (i=0; i<mdb->mi_nattrs; i++) {
600 if ( !ir[i].ir_ai )
601 break;
602 rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi,
603 &ir[i].ir_ai->ai_cursor );
604 if ( rc )
605 return rc;
606 }
607 mdb_tool_ix_id = e->e_id;
608 mdb_tool_ix_txn = txn;
609 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
610 /* Wait for all threads to be ready */
611 while ( mdb_tool_index_tcount ) {
612 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
613 &mdb_tool_index_mutex );
614 }
615
616 for ( i=1; i<mdb_tool_threads; i++ )
617 mdb_tool_index_rec[i].ir_i = LDAP_BUSY;
618 mdb_tool_index_tcount = mdb_tool_threads - 1;
619 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
620 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
621
622 return mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 );
623 } else
624 {
625 return mdb_index_entry_add( op, txn, e );
626 }
627 }
628
629 static int
mdb_tool_index_finish()630 mdb_tool_index_finish()
631 {
632 int i, rc = 0;
633 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
634 for ( i=1; i<mdb_tool_threads; i++ ) {
635 if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) {
636 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
637 &mdb_tool_index_mutex );
638 i--;
639 continue;
640 }
641 if ( mdb_tool_index_rec[i].ir_i ) {
642 rc = mdb_tool_index_rec[i].ir_i;
643 break;
644 }
645 }
646 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
647 return rc;
648 }
649
mdb_tool_entry_put(BackendDB * be,Entry * e,struct berval * text)650 ID mdb_tool_entry_put(
651 BackendDB *be,
652 Entry *e,
653 struct berval *text )
654 {
655 int rc;
656 struct mdb_info *mdb;
657 Operation op = {0};
658 Opheader ohdr = {0};
659
660 assert( be != NULL );
661 assert( slapMode & SLAP_TOOL_MODE );
662
663 assert( text != NULL );
664 assert( text->bv_val != NULL );
665 assert( text->bv_val[0] == '\0' ); /* overconservative? */
666
667 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
668 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn );
669
670 mdb = (struct mdb_info *) be->be_private;
671
672 if ( !mdb_tool_txn ) {
673 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
674 if( rc != 0 ) {
675 snprintf( text->bv_val, text->bv_len,
676 "txn_begin failed: %s (%d)",
677 mdb_strerror(rc), rc );
678 Debug( LDAP_DEBUG_ANY,
679 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
680 text->bv_val );
681 return NOID;
682 }
683 }
684 if ( !idcursor ) {
685 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &idcursor );
686 if( rc != 0 ) {
687 snprintf( text->bv_val, text->bv_len,
688 "cursor_open failed: %s (%d)",
689 mdb_strerror(rc), rc );
690 Debug( LDAP_DEBUG_ANY,
691 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
692 text->bv_val );
693 return NOID;
694 }
695 if ( !mdb->mi_nextid ) {
696 ID dummy;
697 mdb_next_id( be, idcursor, &dummy );
698 }
699 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcp );
700 if( rc != 0 ) {
701 snprintf( text->bv_val, text->bv_len,
702 "cursor_open failed: %s (%d)",
703 mdb_strerror(rc), rc );
704 Debug( LDAP_DEBUG_ANY,
705 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
706 text->bv_val );
707 return NOID;
708 }
709 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcd );
710 if( rc != 0 ) {
711 snprintf( text->bv_val, text->bv_len,
712 "cursor_open failed: %s (%d)",
713 mdb_strerror(rc), rc );
714 Debug( LDAP_DEBUG_ANY,
715 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
716 text->bv_val );
717 return NOID;
718 }
719 }
720
721 op.o_hdr = &ohdr;
722 op.o_bd = be;
723 op.o_tmpmemctx = NULL;
724 op.o_tmpmfuncs = &ch_mfuncs;
725
726 /* add dn2id indices */
727 rc = mdb_tool_next_id( &op, mdb_tool_txn, e, text, 0 );
728 if( rc != 0 ) {
729 goto done;
730 }
731
732 if ( mdb_tool_threads > 1 ) {
733 LDAP_SLIST_INSERT_HEAD( &op.o_extra, &mdb_tool_axinfo[0]->ai_oe, oe_next );
734 }
735 rc = mdb_tool_index_add( &op, mdb_tool_txn, e );
736 if( rc != 0 ) {
737 snprintf( text->bv_val, text->bv_len,
738 "index_entry_add failed: err=%d", rc );
739 Debug( LDAP_DEBUG_ANY,
740 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
741 text->bv_val );
742 goto done;
743 }
744
745
746 /* id2entry index */
747 rc = mdb_id2entry_add( &op, mdb_tool_txn, idcursor, e );
748 if( rc != 0 ) {
749 snprintf( text->bv_val, text->bv_len,
750 "id2entry_add failed: err=%d", rc );
751 Debug( LDAP_DEBUG_ANY,
752 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
753 text->bv_val );
754 goto done;
755 }
756
757 if( mdb->mi_nattrs && mdb_tool_threads > 1 )
758 rc = mdb_tool_index_finish();
759
760 done:
761 if( rc == 0 ) {
762 mdb_writes++;
763 if ( mdb_writes >= mdb_writes_per_commit ) {
764 unsigned i;
765 MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
766 rc = mdb_txn_commit( mdb_tool_txn );
767 for ( i=0; i<mdb->mi_nattrs; i++ )
768 mdb->mi_attrs[i]->ai_cursor = NULL;
769 mdb_writes = 0;
770 mdb_tool_txn = NULL;
771 idcursor = NULL;
772 if( rc != 0 ) {
773 mdb->mi_numads = 0;
774 snprintf( text->bv_val, text->bv_len,
775 "txn_commit failed: %s (%d)",
776 mdb_strerror(rc), rc );
777 Debug( LDAP_DEBUG_ANY,
778 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
779 text->bv_val );
780 e->e_id = NOID;
781 }
782 }
783
784 } else {
785 unsigned i;
786 mdb_txn_abort( mdb_tool_txn );
787 mdb_tool_txn = NULL;
788 idcursor = NULL;
789 for ( i=0; i<mdb->mi_nattrs; i++ )
790 mdb->mi_attrs[i]->ai_cursor = NULL;
791 mdb_writes = 0;
792 snprintf( text->bv_val, text->bv_len,
793 "txn_aborted! %s (%d)",
794 rc == LDAP_OTHER ? "Internal error" :
795 mdb_strerror(rc), rc );
796 Debug( LDAP_DEBUG_ANY,
797 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
798 text->bv_val );
799 e->e_id = NOID;
800 }
801
802 return e->e_id;
803 }
804
805 static int mdb_dn2id_upgrade( BackendDB *be );
806
mdb_tool_entry_reindex(BackendDB * be,ID id,AttributeDescription ** adv)807 int mdb_tool_entry_reindex(
808 BackendDB *be,
809 ID id,
810 AttributeDescription **adv )
811 {
812 struct mdb_info *mi = (struct mdb_info *) be->be_private;
813 int rc;
814 Entry *e;
815 Operation op = {0};
816 Opheader ohdr = {0};
817
818 Debug( LDAP_DEBUG_ARGS,
819 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
820 (long) id );
821 assert( tool_base == NULL );
822 assert( tool_filter == NULL );
823
824 /* Special: do a dn2id upgrade */
825 if ( adv && adv[0] == slap_schema.si_ad_entryDN ) {
826 /* short-circuit tool_entry_next() */
827 mdb_cursor_get( cursor, &key, &data, MDB_LAST );
828 return mdb_dn2id_upgrade( be );
829 }
830
831 /* No indexes configured, nothing to do. Could return an
832 * error here to shortcut things.
833 */
834 if (!mi->mi_attrs) {
835 return 0;
836 }
837
838 /* Check for explicit list of attrs to index */
839 if ( adv ) {
840 int i, j, n;
841
842 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
843 /* count */
844 for ( n = 0; adv[n]; n++ ) ;
845
846 /* insertion sort */
847 for ( i = 0; i < n; i++ ) {
848 AttributeDescription *ad = adv[i];
849 for ( j = i-1; j>=0; j--) {
850 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
851 adv[j+1] = adv[j];
852 }
853 adv[j+1] = ad;
854 }
855 }
856
857 for ( i = 0; adv[i]; i++ ) {
858 if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
859 for ( j = i+1; j < mi->mi_nattrs; j++ ) {
860 if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
861 AttrInfo *ai = mi->mi_attrs[i];
862 mi->mi_attrs[i] = mi->mi_attrs[j];
863 mi->mi_attrs[j] = ai;
864 break;
865 }
866 }
867 if ( j == mi->mi_nattrs ) {
868 Debug( LDAP_DEBUG_ANY,
869 LDAP_XSTRING(mdb_tool_entry_reindex)
870 ": no index configured for %s\n",
871 adv[i]->ad_cname.bv_val );
872 return -1;
873 }
874 }
875 }
876 mi->mi_nattrs = i;
877 }
878
879 e = mdb_tool_entry_get( be, id );
880
881 if( e == NULL ) {
882 Debug( LDAP_DEBUG_ANY,
883 LDAP_XSTRING(mdb_tool_entry_reindex)
884 ": could not locate id=%ld\n",
885 (long) id );
886 return -1;
887 }
888
889 if ( !txi ) {
890 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
891 if( rc != 0 ) {
892 Debug( LDAP_DEBUG_ANY,
893 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
894 "txn_begin failed: %s (%d)\n",
895 mdb_strerror(rc), rc );
896 goto done;
897 }
898 }
899
900 if ( slapMode & SLAP_TRUNCATE_MODE ) {
901 int i;
902 for ( i=0; i < mi->mi_nattrs; i++ ) {
903 rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 );
904 if ( rc ) {
905 Debug( LDAP_DEBUG_ANY,
906 LDAP_XSTRING(mdb_tool_entry_reindex)
907 ": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
908 mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
909 mdb_strerror(rc), rc );
910 return -1;
911 }
912 }
913 slapMode ^= SLAP_TRUNCATE_MODE;
914 }
915
916 /*
917 * just (re)add them for now
918 * Use truncate mode to empty/reset index databases
919 */
920
921 Debug( LDAP_DEBUG_TRACE,
922 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
923 (long) id );
924
925 op.o_hdr = &ohdr;
926 op.o_bd = be;
927 op.o_tmpmemctx = NULL;
928 op.o_tmpmfuncs = &ch_mfuncs;
929
930 rc = mdb_tool_index_add( &op, txi, e );
931
932 done:
933 if( rc == 0 ) {
934 mdb_writes++;
935 if ( mdb_writes >= mdb_writes_per_commit ) {
936 MDB_val key;
937 unsigned i;
938 MDB_TOOL_IDL_FLUSH( be, txi );
939 rc = mdb_txn_commit( txi );
940 mdb_writes = 0;
941 for ( i=0; i<mi->mi_nattrs; i++ )
942 mi->mi_attrs[i]->ai_cursor = NULL;
943 if( rc != 0 ) {
944 Debug( LDAP_DEBUG_ANY,
945 "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
946 ": txn_commit failed: %s (%d)\n",
947 mdb_strerror(rc), rc );
948 e->e_id = NOID;
949 }
950 mdb_cursor_close( cursor );
951 txi = NULL;
952 /* Must close the read txn to allow old pages to be reclaimed. */
953 mdb_txn_abort( mdb_tool_txn );
954 /* and then reopen it so that tool_entry_next still works. */
955 mdb_txn_begin( mi->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
956 mdb_cursor_open( mdb_tool_txn, mi->mi_id2entry, &cursor );
957 key.mv_data = &id;
958 key.mv_size = sizeof(ID);
959 mdb_cursor_get( cursor, &key, NULL, MDB_SET );
960 }
961
962 } else {
963 unsigned i;
964 mdb_writes = 0;
965 mdb_cursor_close( cursor );
966 cursor = NULL;
967 mdb_txn_abort( txi );
968 for ( i=0; i<mi->mi_nattrs; i++ )
969 mi->mi_attrs[i]->ai_cursor = NULL;
970 Debug( LDAP_DEBUG_ANY,
971 "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
972 ": txn_aborted! err=%d\n",
973 rc );
974 e->e_id = NOID;
975 txi = NULL;
976 }
977 mdb_entry_release( &op, e, 0 );
978
979 return rc;
980 }
981
mdb_tool_entry_modify(BackendDB * be,Entry * e,struct berval * text)982 ID mdb_tool_entry_modify(
983 BackendDB *be,
984 Entry *e,
985 struct berval *text )
986 {
987 int rc;
988 struct mdb_info *mdb;
989 Operation op = {0};
990 Opheader ohdr = {0};
991
992 assert( be != NULL );
993 assert( slapMode & SLAP_TOOL_MODE );
994
995 assert( text != NULL );
996 assert( text->bv_val != NULL );
997 assert( text->bv_val[0] == '\0' ); /* overconservative? */
998
999 assert ( e->e_id != NOID );
1000
1001 Debug( LDAP_DEBUG_TRACE,
1002 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
1003 (long) e->e_id, e->e_dn );
1004
1005 mdb = (struct mdb_info *) be->be_private;
1006
1007 if( cursor ) {
1008 mdb_cursor_close( cursor );
1009 cursor = NULL;
1010 }
1011 if ( !mdb_tool_txn ) {
1012 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
1013 if( rc != 0 ) {
1014 snprintf( text->bv_val, text->bv_len,
1015 "txn_begin failed: %s (%d)",
1016 mdb_strerror(rc), rc );
1017 Debug( LDAP_DEBUG_ANY,
1018 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1019 text->bv_val );
1020 return NOID;
1021 }
1022 }
1023
1024 op.o_hdr = &ohdr;
1025 op.o_bd = be;
1026 op.o_tmpmemctx = NULL;
1027 op.o_tmpmfuncs = &ch_mfuncs;
1028
1029 /* id2entry index */
1030 rc = mdb_id2entry_update( &op, mdb_tool_txn, NULL, e );
1031 if( rc != 0 ) {
1032 snprintf( text->bv_val, text->bv_len,
1033 "id2entry_update failed: err=%d", rc );
1034 Debug( LDAP_DEBUG_ANY,
1035 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1036 text->bv_val );
1037 goto done;
1038 }
1039
1040 done:
1041 if( rc == 0 ) {
1042 rc = mdb_txn_commit( mdb_tool_txn );
1043 if( rc != 0 ) {
1044 mdb->mi_numads = 0;
1045 snprintf( text->bv_val, text->bv_len,
1046 "txn_commit failed: %s (%d)",
1047 mdb_strerror(rc), rc );
1048 Debug( LDAP_DEBUG_ANY,
1049 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
1050 "%s\n", text->bv_val );
1051 e->e_id = NOID;
1052 }
1053
1054 } else {
1055 mdb_txn_abort( mdb_tool_txn );
1056 snprintf( text->bv_val, text->bv_len,
1057 "txn_aborted! %s (%d)",
1058 mdb_strerror(rc), rc );
1059 Debug( LDAP_DEBUG_ANY,
1060 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1061 text->bv_val );
1062 e->e_id = NOID;
1063 }
1064 mdb_tool_txn = NULL;
1065
1066 return e->e_id;
1067 }
1068
mdb_tool_entry_delete(BackendDB * be,struct berval * ndn,struct berval * text)1069 int mdb_tool_entry_delete(
1070 BackendDB *be,
1071 struct berval *ndn,
1072 struct berval *text )
1073 {
1074 int rc;
1075 struct mdb_info *mdb;
1076 Operation op = {0};
1077 Opheader ohdr = {0};
1078 Entry *e;
1079
1080 assert( be != NULL );
1081 assert( slapMode & SLAP_TOOL_MODE );
1082
1083 assert( text != NULL );
1084 assert( text->bv_val != NULL );
1085 assert( text->bv_val[0] == '\0' ); /* overconservative? */
1086
1087 assert ( ndn != NULL );
1088 assert ( ndn->bv_val != NULL );
1089
1090 Debug( LDAP_DEBUG_TRACE,
1091 "=> " LDAP_XSTRING(mdb_tool_entry_delete) "( %s )\n",
1092 ndn->bv_val );
1093
1094 mdb = (struct mdb_info *) be->be_private;
1095
1096 assert( cursor == NULL );
1097 if( cursor ) {
1098 mdb_cursor_close( cursor );
1099 cursor = NULL;
1100 }
1101 if( !mdb_tool_txn ) {
1102 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
1103 if( rc != 0 ) {
1104 snprintf( text->bv_val, text->bv_len,
1105 "txn_begin failed: %s (%d)",
1106 mdb_strerror(rc), rc );
1107 Debug( LDAP_DEBUG_ANY,
1108 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1109 text->bv_val );
1110 return LDAP_OTHER;
1111 }
1112 }
1113
1114 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &cursor );
1115 if( rc != 0 ) {
1116 snprintf( text->bv_val, text->bv_len,
1117 "cursor_open failed: %s (%d)",
1118 mdb_strerror(rc), rc );
1119 Debug( LDAP_DEBUG_ANY,
1120 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1121 text->bv_val );
1122 return LDAP_OTHER;
1123 }
1124
1125 op.o_hdr = &ohdr;
1126 op.o_bd = be;
1127 op.o_tmpmemctx = NULL;
1128 op.o_tmpmfuncs = &ch_mfuncs;
1129
1130 rc = mdb_dn2entry( &op, mdb_tool_txn, cursor, ndn, &e, NULL, 0 );
1131 if( rc != 0 ) {
1132 snprintf( text->bv_val, text->bv_len,
1133 "dn2entry failed: %s (%d)",
1134 mdb_strerror(rc), rc );
1135 Debug( LDAP_DEBUG_ANY,
1136 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1137 text->bv_val );
1138 goto done;
1139 }
1140
1141 /* check that we wouldn't orphan any children */
1142 rc = mdb_dn2id_children( &op, mdb_tool_txn, e );
1143 if( rc != MDB_NOTFOUND ) {
1144 switch( rc ) {
1145 case 0:
1146 snprintf( text->bv_val, text->bv_len,
1147 "delete failed:"
1148 " subordinate objects must be deleted first");
1149 break;
1150 default:
1151 snprintf( text->bv_val, text->bv_len,
1152 "has_children failed: %s (%d)",
1153 mdb_strerror(rc), rc );
1154 break;
1155 }
1156 rc = -1;
1157 Debug( LDAP_DEBUG_ANY,
1158 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1159 text->bv_val );
1160 goto done;
1161 }
1162
1163 /* delete from dn2id */
1164 rc = mdb_dn2id_delete( &op, cursor, e->e_id, 1 );
1165 if( rc != 0 ) {
1166 snprintf( text->bv_val, text->bv_len,
1167 "dn2id_delete failed: err=%d", rc );
1168 Debug( LDAP_DEBUG_ANY,
1169 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1170 text->bv_val );
1171 goto done;
1172 }
1173
1174 /* deindex values */
1175 rc = mdb_index_entry_del( &op, mdb_tool_txn, e );
1176 if( rc != 0 ) {
1177 snprintf( text->bv_val, text->bv_len,
1178 "entry_delete failed: err=%d", rc );
1179 Debug( LDAP_DEBUG_ANY,
1180 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1181 text->bv_val );
1182 goto done;
1183 }
1184
1185 /* do the deletion */
1186 rc = mdb_id2entry_delete( be, mdb_tool_txn, e );
1187 if( rc != 0 ) {
1188 snprintf( text->bv_val, text->bv_len,
1189 "id2entry_update failed: err=%d", rc );
1190 Debug( LDAP_DEBUG_ANY,
1191 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1192 text->bv_val );
1193 goto done;
1194 }
1195
1196 done:
1197 /* free entry */
1198 if( e != NULL ) {
1199 mdb_entry_return( &op, e );
1200 }
1201
1202 if( rc == 0 ) {
1203 rc = mdb_txn_commit( mdb_tool_txn );
1204 if( rc != 0 ) {
1205 snprintf( text->bv_val, text->bv_len,
1206 "txn_commit failed: %s (%d)",
1207 mdb_strerror(rc), rc );
1208 Debug( LDAP_DEBUG_ANY,
1209 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": "
1210 "%s\n", text->bv_val );
1211 }
1212
1213 } else {
1214 mdb_txn_abort( mdb_tool_txn );
1215 snprintf( text->bv_val, text->bv_len,
1216 "txn_aborted! %s (%d)",
1217 mdb_strerror(rc), rc );
1218 Debug( LDAP_DEBUG_ANY,
1219 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1220 text->bv_val );
1221 }
1222 mdb_tool_txn = NULL;
1223 cursor = NULL;
1224
1225 return rc;
1226 }
1227
1228 static void *
mdb_tool_index_task(void * ctx,void * ptr)1229 mdb_tool_index_task( void *ctx, void *ptr )
1230 {
1231 int base = *(int *)ptr;
1232 Operation op = {0};
1233 Opheader ohdr = {0};
1234 AttrIxInfo ai = {0}, *aio;
1235
1236 free( ptr );
1237 op.o_hdr = &ohdr;
1238 op.o_bd = mdb_tool_ix_be;
1239 op.o_tmpmemctx = NULL;
1240 op.o_tmpmfuncs = &ch_mfuncs;
1241 aio = mdb_tool_axinfo[base];
1242 mdb_tool_axinfo[base] = &ai;
1243 LDAP_SLIST_INSERT_HEAD( &op.o_extra, &ai.ai_oe, oe_next );
1244 while ( 1 ) {
1245 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
1246 mdb_tool_index_tcount--;
1247 if ( !mdb_tool_index_tcount )
1248 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1249 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
1250 &mdb_tool_index_mutex );
1251 if ( slapd_shutdown ) {
1252 mdb_tool_index_tcount--;
1253 if ( !mdb_tool_index_tcount )
1254 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1255 *aio = ai;
1256 mdb_tool_axinfo[base] = aio;
1257 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1258 break;
1259 }
1260 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1261 mdb_tool_index_rec[base].ir_i = mdb_index_recrun( &op,
1262 mdb_tool_ix_txn,
1263 mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
1264 }
1265
1266 return NULL;
1267 }
1268
1269 #ifdef MDB_TOOL_IDL_CACHING
1270 static int
mdb_tool_idl_cmp(const void * v1,const void * v2)1271 mdb_tool_idl_cmp( const void *v1, const void *v2 )
1272 {
1273 const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
1274 int rc;
1275
1276 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
1277 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
1278 }
1279
1280 static int
mdb_tool_idl_flush_one(MDB_cursor * mc,AttrIxInfo * ai,mdb_tool_idl_cache * ic)1281 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrIxInfo *ai, mdb_tool_idl_cache *ic )
1282 {
1283 mdb_tool_idl_cache_entry *ice;
1284 MDB_val key, data[2];
1285 int i, rc;
1286 ID id, nid;
1287
1288 /* Freshly allocated, ignore it */
1289 if ( !ic->head && ic->count <= MDB_idl_db_size ) {
1290 return 0;
1291 }
1292
1293 key.mv_data = ic->kstr.bv_val;
1294 key.mv_size = ic->kstr.bv_len;
1295
1296 if ( ic->count > MDB_idl_db_size ) {
1297 while ( ic->flags & WAS_FOUND ) {
1298 rc = mdb_cursor_get( mc, &key, data, MDB_SET );
1299 if ( rc ) {
1300 /* FIXME: find out why this happens */
1301 ic->flags = 0;
1302 break;
1303 }
1304 if ( ic->flags & WAS_RANGE ) {
1305 /* Skip lo */
1306 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1307
1308 /* Get hi */
1309 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1310
1311 /* Store range hi */
1312 data[0].mv_data = &ic->last;
1313 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
1314 } else {
1315 /* Delete old data, replace with range */
1316 ic->first = *(ID *)data[0].mv_data;
1317 mdb_cursor_del( mc, MDB_NODUPDATA );
1318 }
1319 break;
1320 }
1321 if ( !(ic->flags & WAS_RANGE)) {
1322 /* range, didn't exist before */
1323 nid = 0;
1324 data[0].mv_size = sizeof(ID);
1325 data[0].mv_data = &nid;
1326 rc = mdb_cursor_put( mc, &key, data, 0 );
1327 if ( rc == 0 ) {
1328 data[0].mv_data = &ic->first;
1329 rc = mdb_cursor_put( mc, &key, data, 0 );
1330 if ( rc == 0 ) {
1331 data[0].mv_data = &ic->last;
1332 rc = mdb_cursor_put( mc, &key, data, 0 );
1333 }
1334 }
1335 if ( rc ) {
1336 rc = -1;
1337 }
1338 }
1339 } else {
1340 /* Normal write */
1341 int n;
1342
1343 data[0].mv_size = sizeof(ID);
1344 rc = 0;
1345 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1346 int end;
1347 if ( ice->next ) {
1348 end = IDBLOCK;
1349 } else {
1350 end = (ic->count-ic->offset) & (IDBLOCK-1);
1351 if ( !end )
1352 end = IDBLOCK;
1353 }
1354 data[1].mv_size = end;
1355 data[0].mv_data = ice->ids;
1356 rc = mdb_cursor_put( mc, &key, data, MDB_APPENDDUP|MDB_MULTIPLE );
1357 if ( rc ) {
1358 rc = -1;
1359 break;
1360 }
1361 }
1362 if ( ic->head ) {
1363 ic->tail->next = ai->ai_flist;
1364 ai->ai_flist = ic->head;
1365 }
1366 }
1367 ic->head = ai->ai_clist;
1368 ai->ai_clist = ic;
1369 return rc;
1370 }
1371
1372 static int
mdb_tool_idl_flush_db(MDB_txn * txn,AttrInfo * ai,AttrIxInfo * ax)1373 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai, AttrIxInfo *ax )
1374 {
1375 MDB_cursor *mc;
1376 Avlnode *root;
1377 int rc;
1378
1379 mdb_cursor_open( txn, ai->ai_dbi, &mc );
1380 root = ldap_tavl_end( ai->ai_root, TAVL_DIR_LEFT );
1381 do {
1382 rc = mdb_tool_idl_flush_one( mc, ax, root->avl_data );
1383 if ( rc != -1 )
1384 rc = 0;
1385 } while ((root = ldap_tavl_next(root, TAVL_DIR_RIGHT)));
1386 mdb_cursor_close( mc );
1387
1388 return rc;
1389 }
1390
1391 static int
mdb_tool_idl_flush(BackendDB * be,MDB_txn * txn)1392 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
1393 {
1394 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
1395 int rc = 0;
1396 unsigned int i, dbi;
1397
1398 for ( i=0; i < mdb->mi_nattrs; i++ ) {
1399 if ( !mdb->mi_attrs[i]->ai_root ) continue;
1400 rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i], mdb_tool_axinfo[i % mdb_tool_threads] );
1401 ldap_tavl_free(mdb->mi_attrs[i]->ai_root, NULL);
1402 mdb->mi_attrs[i]->ai_root = NULL;
1403 if ( rc )
1404 break;
1405 }
1406 return rc;
1407 }
1408
mdb_tool_idl_add(BackendDB * be,MDB_cursor * mc,struct berval * keys,ID id)1409 int mdb_tool_idl_add(
1410 BackendDB *be,
1411 MDB_cursor *mc,
1412 struct berval *keys,
1413 ID id )
1414 {
1415 MDB_dbi dbi;
1416 mdb_tool_idl_cache *ic, itmp;
1417 mdb_tool_idl_cache_entry *ice;
1418 int i, rc, lcount;
1419 AttrIxInfo *ax = (AttrIxInfo *)mc;
1420 AttrInfo *ai = (AttrInfo *)ax->ai_ai;
1421 mc = ai->ai_cursor;
1422
1423 dbi = ai->ai_dbi;
1424 for (i=0; keys[i].bv_val; i++) {
1425 itmp.kstr = keys[i];
1426 ic = ldap_tavl_find( ai->ai_root, &itmp, mdb_tool_idl_cmp );
1427
1428 /* No entry yet, create one */
1429 if ( !ic ) {
1430 MDB_val key, data;
1431 ID nid;
1432 int rc;
1433
1434 if ( ax->ai_clist ) {
1435 ic = ax->ai_clist;
1436 ax->ai_clist = ic->head;
1437 } else {
1438 ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
1439 }
1440 ic->kstr.bv_len = itmp.kstr.bv_len;
1441 ic->kstr.bv_val = (char *)(ic+1);
1442 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1443 ic->head = ic->tail = NULL;
1444 ic->last = 0;
1445 ic->count = 0;
1446 ic->offset = 0;
1447 ic->flags = 0;
1448 ldap_tavl_insert( &ai->ai_root, ic, mdb_tool_idl_cmp,
1449 ldap_avl_dup_error );
1450
1451 /* load existing key count here */
1452 key.mv_size = keys[i].bv_len;
1453 key.mv_data = keys[i].bv_val;
1454 rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
1455 if ( rc == 0 ) {
1456 ic->flags |= WAS_FOUND;
1457 nid = *(ID *)data.mv_data;
1458 if ( nid == 0 ) {
1459 ic->count = MDB_idl_db_size+1;
1460 ic->flags |= WAS_RANGE;
1461 } else {
1462 size_t count;
1463
1464 mdb_cursor_count( mc, &count );
1465 ic->count = count;
1466 ic->first = nid;
1467 ic->offset = count & (IDBLOCK-1);
1468 }
1469 }
1470 }
1471 /* are we a range already? */
1472 if ( ic->count > MDB_idl_db_size ) {
1473 ic->last = id;
1474 continue;
1475 /* Are we at the limit, and converting to a range? */
1476 } else if ( ic->count == MDB_idl_db_size ) {
1477 if ( ic->head ) {
1478 ic->tail->next = ax->ai_flist;
1479 ax->ai_flist = ic->head;
1480 }
1481 ic->head = ic->tail = NULL;
1482 ic->last = id;
1483 ic->count++;
1484 continue;
1485 }
1486 /* No free block, create that too */
1487 lcount = (ic->count-ic->offset) & (IDBLOCK-1);
1488 if ( !ic->tail || lcount == 0) {
1489 if ( ax->ai_flist ) {
1490 ice = ax->ai_flist;
1491 ax->ai_flist = ice->next;
1492 } else {
1493 ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
1494 }
1495 ice->next = NULL;
1496 if ( !ic->head ) {
1497 ic->head = ice;
1498 } else {
1499 ic->tail->next = ice;
1500 }
1501 ic->tail = ice;
1502 if ( lcount )
1503 ice->ids[lcount-1] = 0;
1504 if ( !ic->count )
1505 ic->first = id;
1506 }
1507 ice = ic->tail;
1508 if (!lcount || ice->ids[lcount-1] != id) {
1509 ice->ids[lcount] = id;
1510 ic->count++;
1511 }
1512 }
1513
1514 return 0;
1515 }
1516 #endif /* MDB_TOOL_IDL_CACHING */
1517
1518 /* Upgrade from pre 2.4.34 dn2id format */
1519
1520 #include <ac/unistd.h>
1521 #include <lutil_meter.h>
1522
1523 #define STACKSIZ 2048
1524
1525 typedef struct rec {
1526 ID id;
1527 size_t len;
1528 char rdn[512];
1529 } rec;
1530
1531 static int
mdb_dn2id_upgrade(BackendDB * be)1532 mdb_dn2id_upgrade( BackendDB *be ) {
1533 struct mdb_info *mi = (struct mdb_info *) be->be_private;
1534 MDB_txn *mt;
1535 MDB_cursor *mc = NULL;
1536 MDB_val key, data;
1537 int rc, writes=0, depth=0;
1538 int enable_meter = 0;
1539 ID id = 0, *num, count = 0;
1540 rec *stack;
1541 lutil_meter_t meter;
1542
1543 if (!(mi->mi_flags & MDB_NEED_UPGRADE)) {
1544 Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n",
1545 be->be_suffix[0].bv_val );
1546 return 0;
1547 }
1548
1549 {
1550 MDB_stat st;
1551
1552 mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st);
1553 if (!st.ms_entries) {
1554 /* Empty DB, nothing to upgrade? */
1555 return 0;
1556 }
1557 if (isatty(2))
1558 enable_meter = !lutil_meter_open(&meter,
1559 &lutil_meter_text_display,
1560 &lutil_meter_linear_estimator,
1561 st.ms_entries);
1562 }
1563
1564 num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec)));
1565 stack = (rec *)(num + STACKSIZ);
1566
1567 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1568 if (rc) {
1569 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n",
1570 mdb_strerror(rc), rc );
1571 goto leave;
1572 }
1573 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1574 if (rc) {
1575 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n",
1576 mdb_strerror(rc), rc );
1577 goto leave;
1578 }
1579
1580 key.mv_size = sizeof(ID);
1581 /* post-order depth-first update */
1582 for(;;) {
1583 size_t dkids;
1584 unsigned char *ptr;
1585
1586 /* visit */
1587 key.mv_data = &id;
1588 stack[depth].id = id;
1589 rc = mdb_cursor_get(mc, &key, &data, MDB_SET);
1590 if (rc) {
1591 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n",
1592 mdb_strerror(rc), rc );
1593 goto leave;
1594 }
1595 num[depth] = 1;
1596
1597 rc = mdb_cursor_count(mc, &dkids);
1598 if (rc) {
1599 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n",
1600 mdb_strerror(rc), rc );
1601 goto leave;
1602 }
1603 if (dkids > 1) {
1604 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1605 down:
1606 ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID);
1607 memcpy(&id, ptr, sizeof(ID));
1608 depth++;
1609 memcpy(stack[depth].rdn, data.mv_data, data.mv_size);
1610 stack[depth].len = data.mv_size;
1611 continue;
1612 }
1613
1614
1615 /* pop: write updated count, advance to next node */
1616 pop:
1617 /* update superior counts */
1618 if (depth)
1619 num[depth-1] += num[depth];
1620
1621 key.mv_data = &id;
1622 id = stack[depth-1].id;
1623 data.mv_data = stack[depth].rdn;
1624 data.mv_size = stack[depth].len;
1625 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1626 if (rc) {
1627 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n",
1628 mdb_strerror(rc), rc );
1629 goto leave;
1630 }
1631 data.mv_data = stack[depth].rdn;
1632 ptr = (unsigned char *)data.mv_data + data.mv_size;
1633 memcpy(ptr, &num[depth], sizeof(ID));
1634 data.mv_size += sizeof(ID);
1635 rc = mdb_cursor_del(mc, 0);
1636 if (rc) {
1637 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n",
1638 mdb_strerror(rc), rc );
1639 goto leave;
1640 }
1641 rc = mdb_cursor_put(mc, &key, &data, 0);
1642 if (rc) {
1643 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n",
1644 mdb_strerror(rc), rc );
1645 goto leave;
1646 }
1647 count++;
1648 #if 1
1649 if (enable_meter)
1650 lutil_meter_update(&meter, count, 0);
1651 #else
1652 {
1653 int len;
1654 ptr = data.mv_data;
1655 len = (ptr[0] & 0x7f) << 8 | ptr[1];
1656 printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2);
1657 }
1658 #endif
1659 writes++;
1660 if (writes == 1000) {
1661 mdb_cursor_close(mc);
1662 rc = mdb_txn_commit(mt);
1663 if (rc) {
1664 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n",
1665 mdb_strerror(rc), rc );
1666 goto leave;
1667 }
1668 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1669 if (rc) {
1670 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n",
1671 mdb_strerror(rc), rc );
1672 goto leave;
1673 }
1674 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1675 if (rc) {
1676 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n",
1677 mdb_strerror(rc), rc );
1678 goto leave;
1679 }
1680 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1681 if (rc) {
1682 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n",
1683 mdb_strerror(rc), rc );
1684 goto leave;
1685 }
1686 writes = 0;
1687 }
1688 depth--;
1689
1690 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1691 if (rc == 0)
1692 goto down;
1693 rc = 0;
1694 if (depth)
1695 goto pop;
1696 else
1697 break;
1698 }
1699 leave:
1700 mdb_cursor_close(mc);
1701 if (mt) {
1702 int r2;
1703 r2 = mdb_txn_commit(mt);
1704 if (r2) {
1705 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n",
1706 mdb_strerror(r2), r2 );
1707 if (!rc)
1708 rc = r2;
1709 }
1710 }
1711 ch_free(num);
1712 if (enable_meter) {
1713 lutil_meter_update(&meter, count, 1);
1714 lutil_meter_close(&meter);
1715 }
1716 return rc;
1717 }
1718