1 /* $NetBSD: tools.c,v 1.3 2021/08/14 16:15:00 christos Exp $ */ 2 3 /* tools.c - tools for slap tools */ 4 /* $OpenLDAP$ */ 5 /* This work is part of OpenLDAP Software <http://www.openldap.org/>. 6 * 7 * Copyright 2011-2021 The OpenLDAP Foundation. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted only as authorized by the OpenLDAP 12 * Public License. 13 * 14 * A copy of this license is available in the file LICENSE in the 15 * top-level directory of the distribution or, alternatively, at 16 * <http://www.OpenLDAP.org/license.html>. 17 */ 18 19 #include <sys/cdefs.h> 20 __RCSID("$NetBSD: tools.c,v 1.3 2021/08/14 16:15:00 christos Exp $"); 21 22 #include "portable.h" 23 24 #include <stdio.h> 25 #include <ac/string.h> 26 #include <ac/errno.h> 27 28 #define AVL_INTERNAL 29 #include "back-mdb.h" 30 #include "idl.h" 31 32 #ifdef MDB_TOOL_IDL_CACHING 33 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn ); 34 35 #define IDBLOCK 1024 36 37 typedef struct mdb_tool_idl_cache_entry { 38 struct mdb_tool_idl_cache_entry *next; 39 ID ids[IDBLOCK]; 40 } mdb_tool_idl_cache_entry; 41 42 typedef struct mdb_tool_idl_cache { 43 struct berval kstr; 44 mdb_tool_idl_cache_entry *head, *tail; 45 ID first, last; 46 int count; 47 short offset; 48 short flags; 49 } mdb_tool_idl_cache; 50 #define WAS_FOUND 0x01 51 #define WAS_RANGE 0x02 52 53 #define MDB_TOOL_IDL_FLUSH(be, txn) mdb_tool_idl_flush(be, txn) 54 #else 55 #define MDB_TOOL_IDL_FLUSH(be, txn) 56 #endif /* MDB_TOOL_IDL_CACHING */ 57 58 MDB_txn *mdb_tool_txn = NULL; 59 60 static MDB_txn *txi = NULL; 61 static MDB_cursor *cursor = NULL, *idcursor = NULL; 62 static MDB_cursor *mcp = NULL, *mcd = NULL; 63 static MDB_val key, data; 64 static ID previd = NOID; 65 66 typedef struct dn_id { 67 ID id; 68 struct berval dn; 69 } dn_id; 70 71 #define HOLE_SIZE 4096 72 static dn_id hbuf[HOLE_SIZE], *holes = hbuf; 73 static unsigned nhmax = HOLE_SIZE; 74 static unsigned nholes; 75 76 static struct berval *tool_base; 77 static int tool_scope; 78 static Filter *tool_filter; 79 static Entry *tool_next_entry; 80 81 static ID mdb_tool_ix_id; 82 static BackendDB *mdb_tool_ix_be; 83 static MDB_txn *mdb_tool_ix_txn; 84 static int mdb_tool_index_tcount, mdb_tool_threads; 85 static IndexRec *mdb_tool_index_rec; 86 static AttrIxInfo **mdb_tool_axinfo; 87 static struct mdb_info *mdb_tool_info; 88 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex; 89 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main; 90 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work; 91 static void * mdb_tool_index_task( void *ctx, void *ptr ); 92 93 static int mdb_writes, mdb_writes_per_commit; 94 95 /* Number of ops per commit in Quick mode. 96 * Batching speeds writes overall, but too large a 97 * batch will fail with MDB_TXN_FULL. 98 */ 99 #ifndef MDB_WRITES_PER_COMMIT 100 #define MDB_WRITES_PER_COMMIT 500 101 #endif 102 103 static int 104 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ); 105 106 int mdb_tool_entry_open( 107 BackendDB *be, int mode ) 108 { 109 /* In Quick mode, commit once per 500 entries */ 110 mdb_writes = 0; 111 if ( slapMode & SLAP_TOOL_QUICK ) 112 mdb_writes_per_commit = MDB_WRITES_PER_COMMIT; 113 else 114 mdb_writes_per_commit = 1; 115 116 #ifdef MDB_TOOL_IDL_CACHING /* threaded indexing has no performance advantage */ 117 /* Set up for threaded slapindex */ 118 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) { 119 if ( !mdb_tool_info ) { 120 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 121 ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex ); 122 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main ); 123 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work ); 124 if ( mdb->mi_nattrs ) { 125 int i; 126 mdb_tool_threads = slap_tool_thread_max - 1; 127 if ( mdb_tool_threads > 1 ) { 128 mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec )); 129 mdb_tool_axinfo = ch_calloc( mdb_tool_threads, sizeof( AttrIxInfo* ) + 130 sizeof( AttrIxInfo )); 131 mdb_tool_axinfo[0] = (AttrIxInfo *)(mdb_tool_axinfo + mdb_tool_threads); 132 for (i=1; i<mdb_tool_threads; i++) 133 mdb_tool_axinfo[i] = mdb_tool_axinfo[i-1]+1; 134 mdb_tool_index_tcount = mdb_tool_threads - 1; 135 mdb_tool_ix_be = be; 136 for (i=1; i<mdb_tool_threads; i++) { 137 int *ptr = ch_malloc( sizeof( int )); 138 *ptr = i; 139 ldap_pvt_thread_pool_submit( &connection_pool, 140 mdb_tool_index_task, ptr ); 141 } 142 mdb_tool_info = mdb; 143 } 144 } 145 } 146 } 147 #endif 148 149 return 0; 150 } 151 152 int mdb_tool_entry_close( 153 BackendDB *be ) 154 { 155 #ifdef MDB_TOOL_IDL_CACHING 156 if ( mdb_tool_info ) { 157 int i; 158 slapd_shutdown = 1; 159 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 160 161 /* There might still be some threads starting */ 162 while ( mdb_tool_index_tcount > 0 ) { 163 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 164 &mdb_tool_index_mutex ); 165 } 166 167 mdb_tool_index_tcount = mdb_tool_threads - 1; 168 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work ); 169 170 /* Make sure all threads are stopped */ 171 while ( mdb_tool_index_tcount > 0 ) { 172 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 173 &mdb_tool_index_mutex ); 174 } 175 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 176 177 mdb_tool_info = NULL; 178 slapd_shutdown = 0; 179 ch_free( mdb_tool_index_rec ); 180 mdb_tool_index_tcount = mdb_tool_threads - 1; 181 if (mdb_tool_txn) 182 MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn ); 183 for (i=0; i<mdb_tool_threads; i++) { 184 mdb_tool_idl_cache *ic; 185 mdb_tool_idl_cache_entry *ice; 186 while ((ic = mdb_tool_axinfo[i]->ai_clist)) { 187 mdb_tool_axinfo[i]->ai_clist = ic->head; 188 free(ic); 189 } 190 while ((ice = mdb_tool_axinfo[i]->ai_flist)) { 191 mdb_tool_axinfo[i]->ai_flist = ice->next; 192 free(ice); 193 } 194 } 195 } 196 #endif 197 198 if( idcursor ) { 199 mdb_cursor_close( idcursor ); 200 idcursor = NULL; 201 } 202 if( cursor ) { 203 mdb_cursor_close( cursor ); 204 cursor = NULL; 205 } 206 { 207 struct mdb_info *mdb = be->be_private; 208 if ( mdb ) { 209 int i; 210 for (i=0; i<mdb->mi_nattrs; i++) 211 mdb->mi_attrs[i]->ai_cursor = NULL; 212 } 213 } 214 if( mdb_tool_txn ) { 215 int rc; 216 if (( rc = mdb_txn_commit( mdb_tool_txn ))) { 217 Debug( LDAP_DEBUG_ANY, 218 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: " 219 "txn_commit failed: %s (%d)\n", 220 be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); 221 return -1; 222 } 223 mdb_tool_txn = NULL; 224 } 225 if( txi ) { 226 int rc; 227 if (( rc = mdb_txn_commit( txi ))) { 228 Debug( LDAP_DEBUG_ANY, 229 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: " 230 "txn_commit failed: %s (%d)\n", 231 be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); 232 return -1; 233 } 234 txi = NULL; 235 } 236 237 if( nholes ) { 238 unsigned i; 239 fprintf( stderr, "Error, entries missing!\n"); 240 for (i=0; i<nholes; i++) { 241 fprintf(stderr, " entry %ld: %s\n", 242 holes[i].id, holes[i].dn.bv_val); 243 } 244 nholes = 0; 245 return -1; 246 } 247 248 return 0; 249 } 250 251 ID 252 mdb_tool_entry_first_x( 253 BackendDB *be, 254 struct berval *base, 255 int scope, 256 Filter *f ) 257 { 258 tool_base = base; 259 tool_scope = scope; 260 tool_filter = f; 261 262 return mdb_tool_entry_next( be ); 263 } 264 265 ID mdb_tool_entry_next( 266 BackendDB *be ) 267 { 268 int rc; 269 ID id; 270 struct mdb_info *mdb; 271 272 assert( be != NULL ); 273 assert( slapMode & SLAP_TOOL_MODE ); 274 275 mdb = (struct mdb_info *) be->be_private; 276 assert( mdb != NULL ); 277 278 if ( !mdb_tool_txn ) { 279 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn ); 280 if ( rc ) 281 return NOID; 282 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor ); 283 if ( rc ) { 284 mdb_txn_abort( mdb_tool_txn ); 285 return NOID; 286 } 287 } 288 289 next:; 290 rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT ); 291 292 if( rc ) { 293 return NOID; 294 } 295 296 previd = *(ID *)key.mv_data; 297 id = previd; 298 299 if ( !data.mv_size ) 300 goto next; 301 302 if ( tool_filter || tool_base ) { 303 static Operation op = {0}; 304 static Opheader ohdr = {0}; 305 306 op.o_hdr = &ohdr; 307 op.o_bd = be; 308 op.o_tmpmemctx = NULL; 309 op.o_tmpmfuncs = &ch_mfuncs; 310 311 if ( tool_next_entry ) { 312 mdb_entry_release( &op, tool_next_entry, 0 ); 313 tool_next_entry = NULL; 314 } 315 316 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry ); 317 if ( rc == LDAP_NO_SUCH_OBJECT ) { 318 goto next; 319 } 320 321 assert( tool_next_entry != NULL ); 322 323 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE ) 324 { 325 mdb_entry_release( &op, tool_next_entry, 0 ); 326 tool_next_entry = NULL; 327 goto next; 328 } 329 } 330 331 return id; 332 } 333 334 ID mdb_tool_dn2id_get( 335 Backend *be, 336 struct berval *dn 337 ) 338 { 339 struct mdb_info *mdb; 340 Operation op = {0}; 341 Opheader ohdr = {0}; 342 ID id; 343 int rc; 344 345 if ( BER_BVISEMPTY(dn) ) 346 return 0; 347 348 mdb = (struct mdb_info *) be->be_private; 349 350 if ( !mdb_tool_txn ) { 351 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ? 352 MDB_RDONLY : 0, &mdb_tool_txn ); 353 if ( rc ) 354 return NOID; 355 } 356 357 op.o_hdr = &ohdr; 358 op.o_bd = be; 359 op.o_tmpmemctx = NULL; 360 op.o_tmpmfuncs = &ch_mfuncs; 361 362 rc = mdb_dn2id( &op, mdb_tool_txn, NULL, dn, &id, NULL, NULL, NULL ); 363 if ( rc == MDB_NOTFOUND ) 364 return NOID; 365 366 return id; 367 } 368 369 static int 370 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ) 371 { 372 Operation op = {0}; 373 Opheader ohdr = {0}; 374 375 Entry *e = NULL; 376 struct berval dn = BER_BVNULL, ndn = BER_BVNULL; 377 int rc; 378 379 assert( be != NULL ); 380 assert( slapMode & SLAP_TOOL_MODE ); 381 382 if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) { 383 *ep = tool_next_entry; 384 tool_next_entry = NULL; 385 return LDAP_SUCCESS; 386 } 387 388 if ( id != previd ) { 389 key.mv_size = sizeof(ID); 390 key.mv_data = &id; 391 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET ); 392 if ( rc ) { 393 rc = LDAP_OTHER; 394 goto done; 395 } 396 } 397 if ( !data.mv_size ) { 398 rc = LDAP_NO_SUCH_OBJECT; 399 goto done; 400 } 401 402 op.o_hdr = &ohdr; 403 op.o_bd = be; 404 op.o_tmpmemctx = NULL; 405 op.o_tmpmfuncs = &ch_mfuncs; 406 if ( slapMode & SLAP_TOOL_READONLY ) { 407 rc = mdb_id2name( &op, mdb_tool_txn, &idcursor, id, &dn, &ndn ); 408 if ( rc ) { 409 rc = LDAP_OTHER; 410 goto done; 411 } 412 if ( tool_base != NULL ) { 413 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) { 414 ch_free( dn.bv_val ); 415 ch_free( ndn.bv_val ); 416 rc = LDAP_NO_SUCH_OBJECT; 417 goto done; 418 } 419 } 420 } 421 rc = mdb_entry_decode( &op, mdb_tool_txn, &data, id, &e ); 422 e->e_id = id; 423 if ( !BER_BVISNULL( &dn )) { 424 e->e_name = dn; 425 e->e_nname = ndn; 426 } else { 427 e->e_name.bv_val = NULL; 428 e->e_nname.bv_val = NULL; 429 } 430 431 done: 432 if ( e != NULL ) { 433 *ep = e; 434 } 435 436 return rc; 437 } 438 439 Entry* 440 mdb_tool_entry_get( BackendDB *be, ID id ) 441 { 442 Entry *e = NULL; 443 int rc; 444 445 if ( !mdb_tool_txn ) { 446 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 447 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 448 (slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &mdb_tool_txn ); 449 if ( rc ) 450 return NULL; 451 } 452 if ( !cursor ) { 453 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 454 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor ); 455 if ( rc ) { 456 mdb_txn_abort( mdb_tool_txn ); 457 mdb_tool_txn = NULL; 458 return NULL; 459 } 460 } 461 (void)mdb_tool_entry_get_int( be, id, &e ); 462 return e; 463 } 464 465 static int mdb_tool_next_id( 466 Operation *op, 467 MDB_txn *tid, 468 Entry *e, 469 struct berval *text, 470 int hole ) 471 { 472 struct berval dn = e->e_name; 473 struct berval ndn = e->e_nname; 474 struct berval pdn, npdn, nmatched; 475 ID id, pid = 0; 476 int rc; 477 478 if (ndn.bv_len == 0) { 479 e->e_id = 0; 480 return 0; 481 } 482 483 rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, NULL, &nmatched ); 484 if ( rc == MDB_NOTFOUND ) { 485 if ( !be_issuffix( op->o_bd, &ndn ) ) { 486 ID eid = e->e_id; 487 dnParent( &ndn, &npdn ); 488 if ( nmatched.bv_len != npdn.bv_len ) { 489 dnParent( &dn, &pdn ); 490 e->e_name = pdn; 491 e->e_nname = npdn; 492 rc = mdb_tool_next_id( op, tid, e, text, 1 ); 493 e->e_name = dn; 494 e->e_nname = ndn; 495 if ( rc ) { 496 return rc; 497 } 498 /* If parent didn't exist, it was created just now 499 * and its ID is now in e->e_id. Make sure the current 500 * entry gets added under the new parent ID. 501 */ 502 if ( eid != e->e_id ) { 503 pid = e->e_id; 504 } 505 } else { 506 pid = id; 507 } 508 } 509 rc = mdb_next_id( op->o_bd, idcursor, &e->e_id ); 510 if ( rc ) { 511 snprintf( text->bv_val, text->bv_len, 512 "next_id failed: %s (%d)", 513 mdb_strerror(rc), rc ); 514 Debug( LDAP_DEBUG_ANY, 515 "=> mdb_tool_next_id: %s\n", text->bv_val ); 516 return rc; 517 } 518 rc = mdb_dn2id_add( op, mcp, mcd, pid, 1, 1, e ); 519 if ( rc ) { 520 snprintf( text->bv_val, text->bv_len, 521 "dn2id_add failed: %s (%d)", 522 mdb_strerror(rc), rc ); 523 Debug( LDAP_DEBUG_ANY, 524 "=> mdb_tool_next_id: %s\n", text->bv_val ); 525 } else if ( hole ) { 526 MDB_val key, data; 527 if ( nholes == nhmax - 1 ) { 528 if ( holes == hbuf ) { 529 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 ); 530 AC_MEMCPY( holes, hbuf, sizeof(hbuf) ); 531 } else { 532 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 ); 533 } 534 nhmax *= 2; 535 } 536 ber_dupbv( &holes[nholes].dn, &ndn ); 537 holes[nholes++].id = e->e_id; 538 key.mv_size = sizeof(ID); 539 key.mv_data = &e->e_id; 540 data.mv_size = 0; 541 data.mv_data = NULL; 542 rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE ); 543 if ( rc == MDB_KEYEXIST ) 544 rc = 0; 545 if ( rc ) { 546 snprintf( text->bv_val, text->bv_len, 547 "dummy id2entry add failed: %s (%d)", 548 mdb_strerror(rc), rc ); 549 Debug( LDAP_DEBUG_ANY, 550 "=> mdb_tool_next_id: %s\n", text->bv_val ); 551 } 552 } 553 } else if ( !hole ) { 554 unsigned i, j; 555 556 e->e_id = id; 557 558 for ( i=0; i<nholes; i++) { 559 if ( holes[i].id == e->e_id ) { 560 free(holes[i].dn.bv_val); 561 for (j=i;j<nholes;j++) holes[j] = holes[j+1]; 562 holes[j].id = 0; 563 nholes--; 564 break; 565 } else if ( holes[i].id > e->e_id ) { 566 break; 567 } 568 } 569 } 570 return rc; 571 } 572 573 static int 574 mdb_tool_index_add( 575 Operation *op, 576 MDB_txn *txn, 577 Entry *e ) 578 { 579 struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; 580 581 if ( !mdb->mi_nattrs ) 582 return 0; 583 584 if ( mdb_tool_threads > 1 ) { 585 IndexRec *ir; 586 int i, rc; 587 Attribute *a; 588 589 ir = mdb_tool_index_rec; 590 for (i=0; i<mdb->mi_nattrs; i++) 591 ir[i].ir_attrs = NULL; 592 593 for ( a = e->e_attrs; a != NULL; a = a->a_next ) { 594 rc = mdb_index_recset( mdb, a, a->a_desc->ad_type, 595 &a->a_desc->ad_tags, ir ); 596 if ( rc ) 597 return rc; 598 } 599 for (i=0; i<mdb->mi_nattrs; i++) { 600 if ( !ir[i].ir_ai ) 601 break; 602 rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi, 603 &ir[i].ir_ai->ai_cursor ); 604 if ( rc ) 605 return rc; 606 } 607 mdb_tool_ix_id = e->e_id; 608 mdb_tool_ix_txn = txn; 609 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 610 /* Wait for all threads to be ready */ 611 while ( mdb_tool_index_tcount ) { 612 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 613 &mdb_tool_index_mutex ); 614 } 615 616 for ( i=1; i<mdb_tool_threads; i++ ) 617 mdb_tool_index_rec[i].ir_i = LDAP_BUSY; 618 mdb_tool_index_tcount = mdb_tool_threads - 1; 619 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work ); 620 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 621 622 return mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 ); 623 } else 624 { 625 return mdb_index_entry_add( op, txn, e ); 626 } 627 } 628 629 static int 630 mdb_tool_index_finish() 631 { 632 int i, rc = 0; 633 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 634 for ( i=1; i<mdb_tool_threads; i++ ) { 635 if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) { 636 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 637 &mdb_tool_index_mutex ); 638 i--; 639 continue; 640 } 641 if ( mdb_tool_index_rec[i].ir_i ) { 642 rc = mdb_tool_index_rec[i].ir_i; 643 break; 644 } 645 } 646 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 647 return rc; 648 } 649 650 ID mdb_tool_entry_put( 651 BackendDB *be, 652 Entry *e, 653 struct berval *text ) 654 { 655 int rc; 656 struct mdb_info *mdb; 657 Operation op = {0}; 658 Opheader ohdr = {0}; 659 660 assert( be != NULL ); 661 assert( slapMode & SLAP_TOOL_MODE ); 662 663 assert( text != NULL ); 664 assert( text->bv_val != NULL ); 665 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 666 667 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put) 668 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn ); 669 670 mdb = (struct mdb_info *) be->be_private; 671 672 if ( !mdb_tool_txn ) { 673 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn ); 674 if( rc != 0 ) { 675 snprintf( text->bv_val, text->bv_len, 676 "txn_begin failed: %s (%d)", 677 mdb_strerror(rc), rc ); 678 Debug( LDAP_DEBUG_ANY, 679 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 680 text->bv_val ); 681 return NOID; 682 } 683 } 684 if ( !idcursor ) { 685 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &idcursor ); 686 if( rc != 0 ) { 687 snprintf( text->bv_val, text->bv_len, 688 "cursor_open failed: %s (%d)", 689 mdb_strerror(rc), rc ); 690 Debug( LDAP_DEBUG_ANY, 691 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 692 text->bv_val ); 693 return NOID; 694 } 695 if ( !mdb->mi_nextid ) { 696 ID dummy; 697 mdb_next_id( be, idcursor, &dummy ); 698 } 699 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcp ); 700 if( rc != 0 ) { 701 snprintf( text->bv_val, text->bv_len, 702 "cursor_open failed: %s (%d)", 703 mdb_strerror(rc), rc ); 704 Debug( LDAP_DEBUG_ANY, 705 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 706 text->bv_val ); 707 return NOID; 708 } 709 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcd ); 710 if( rc != 0 ) { 711 snprintf( text->bv_val, text->bv_len, 712 "cursor_open failed: %s (%d)", 713 mdb_strerror(rc), rc ); 714 Debug( LDAP_DEBUG_ANY, 715 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 716 text->bv_val ); 717 return NOID; 718 } 719 } 720 721 op.o_hdr = &ohdr; 722 op.o_bd = be; 723 op.o_tmpmemctx = NULL; 724 op.o_tmpmfuncs = &ch_mfuncs; 725 726 /* add dn2id indices */ 727 rc = mdb_tool_next_id( &op, mdb_tool_txn, e, text, 0 ); 728 if( rc != 0 ) { 729 goto done; 730 } 731 732 if ( mdb_tool_threads > 1 ) { 733 LDAP_SLIST_INSERT_HEAD( &op.o_extra, &mdb_tool_axinfo[0]->ai_oe, oe_next ); 734 } 735 rc = mdb_tool_index_add( &op, mdb_tool_txn, e ); 736 if( rc != 0 ) { 737 snprintf( text->bv_val, text->bv_len, 738 "index_entry_add failed: err=%d", rc ); 739 Debug( LDAP_DEBUG_ANY, 740 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 741 text->bv_val ); 742 goto done; 743 } 744 745 746 /* id2entry index */ 747 rc = mdb_id2entry_add( &op, mdb_tool_txn, idcursor, e ); 748 if( rc != 0 ) { 749 snprintf( text->bv_val, text->bv_len, 750 "id2entry_add failed: err=%d", rc ); 751 Debug( LDAP_DEBUG_ANY, 752 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 753 text->bv_val ); 754 goto done; 755 } 756 757 if( mdb->mi_nattrs && mdb_tool_threads > 1 ) 758 rc = mdb_tool_index_finish(); 759 760 done: 761 if( rc == 0 ) { 762 mdb_writes++; 763 if ( mdb_writes >= mdb_writes_per_commit ) { 764 unsigned i; 765 MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn ); 766 rc = mdb_txn_commit( mdb_tool_txn ); 767 for ( i=0; i<mdb->mi_nattrs; i++ ) 768 mdb->mi_attrs[i]->ai_cursor = NULL; 769 mdb_writes = 0; 770 mdb_tool_txn = NULL; 771 idcursor = NULL; 772 if( rc != 0 ) { 773 mdb->mi_numads = 0; 774 snprintf( text->bv_val, text->bv_len, 775 "txn_commit failed: %s (%d)", 776 mdb_strerror(rc), rc ); 777 Debug( LDAP_DEBUG_ANY, 778 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 779 text->bv_val ); 780 e->e_id = NOID; 781 } 782 } 783 784 } else { 785 unsigned i; 786 mdb_txn_abort( mdb_tool_txn ); 787 mdb_tool_txn = NULL; 788 idcursor = NULL; 789 for ( i=0; i<mdb->mi_nattrs; i++ ) 790 mdb->mi_attrs[i]->ai_cursor = NULL; 791 mdb_writes = 0; 792 snprintf( text->bv_val, text->bv_len, 793 "txn_aborted! %s (%d)", 794 rc == LDAP_OTHER ? "Internal error" : 795 mdb_strerror(rc), rc ); 796 Debug( LDAP_DEBUG_ANY, 797 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 798 text->bv_val ); 799 e->e_id = NOID; 800 } 801 802 return e->e_id; 803 } 804 805 static int mdb_dn2id_upgrade( BackendDB *be ); 806 807 int mdb_tool_entry_reindex( 808 BackendDB *be, 809 ID id, 810 AttributeDescription **adv ) 811 { 812 struct mdb_info *mi = (struct mdb_info *) be->be_private; 813 int rc; 814 Entry *e; 815 Operation op = {0}; 816 Opheader ohdr = {0}; 817 818 Debug( LDAP_DEBUG_ARGS, 819 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n", 820 (long) id ); 821 assert( tool_base == NULL ); 822 assert( tool_filter == NULL ); 823 824 /* Special: do a dn2id upgrade */ 825 if ( adv && adv[0] == slap_schema.si_ad_entryDN ) { 826 /* short-circuit tool_entry_next() */ 827 mdb_cursor_get( cursor, &key, &data, MDB_LAST ); 828 return mdb_dn2id_upgrade( be ); 829 } 830 831 /* No indexes configured, nothing to do. Could return an 832 * error here to shortcut things. 833 */ 834 if (!mi->mi_attrs) { 835 return 0; 836 } 837 838 /* Check for explicit list of attrs to index */ 839 if ( adv ) { 840 int i, j, n; 841 842 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) { 843 /* count */ 844 for ( n = 0; adv[n]; n++ ) ; 845 846 /* insertion sort */ 847 for ( i = 0; i < n; i++ ) { 848 AttributeDescription *ad = adv[i]; 849 for ( j = i-1; j>=0; j--) { 850 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break; 851 adv[j+1] = adv[j]; 852 } 853 adv[j+1] = ad; 854 } 855 } 856 857 for ( i = 0; adv[i]; i++ ) { 858 if ( mi->mi_attrs[i]->ai_desc != adv[i] ) { 859 for ( j = i+1; j < mi->mi_nattrs; j++ ) { 860 if ( mi->mi_attrs[j]->ai_desc == adv[i] ) { 861 AttrInfo *ai = mi->mi_attrs[i]; 862 mi->mi_attrs[i] = mi->mi_attrs[j]; 863 mi->mi_attrs[j] = ai; 864 break; 865 } 866 } 867 if ( j == mi->mi_nattrs ) { 868 Debug( LDAP_DEBUG_ANY, 869 LDAP_XSTRING(mdb_tool_entry_reindex) 870 ": no index configured for %s\n", 871 adv[i]->ad_cname.bv_val ); 872 return -1; 873 } 874 } 875 } 876 mi->mi_nattrs = i; 877 } 878 879 e = mdb_tool_entry_get( be, id ); 880 881 if( e == NULL ) { 882 Debug( LDAP_DEBUG_ANY, 883 LDAP_XSTRING(mdb_tool_entry_reindex) 884 ": could not locate id=%ld\n", 885 (long) id ); 886 return -1; 887 } 888 889 if ( !txi ) { 890 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi ); 891 if( rc != 0 ) { 892 Debug( LDAP_DEBUG_ANY, 893 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": " 894 "txn_begin failed: %s (%d)\n", 895 mdb_strerror(rc), rc ); 896 goto done; 897 } 898 } 899 900 if ( slapMode & SLAP_TRUNCATE_MODE ) { 901 int i; 902 for ( i=0; i < mi->mi_nattrs; i++ ) { 903 rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 ); 904 if ( rc ) { 905 Debug( LDAP_DEBUG_ANY, 906 LDAP_XSTRING(mdb_tool_entry_reindex) 907 ": (Truncate) mdb_drop(%s) failed: %s (%d)\n", 908 mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val, 909 mdb_strerror(rc), rc ); 910 return -1; 911 } 912 } 913 slapMode ^= SLAP_TRUNCATE_MODE; 914 } 915 916 /* 917 * just (re)add them for now 918 * Use truncate mode to empty/reset index databases 919 */ 920 921 Debug( LDAP_DEBUG_TRACE, 922 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n", 923 (long) id ); 924 925 op.o_hdr = &ohdr; 926 op.o_bd = be; 927 op.o_tmpmemctx = NULL; 928 op.o_tmpmfuncs = &ch_mfuncs; 929 930 rc = mdb_tool_index_add( &op, txi, e ); 931 932 done: 933 if( rc == 0 ) { 934 mdb_writes++; 935 if ( mdb_writes >= mdb_writes_per_commit ) { 936 MDB_val key; 937 unsigned i; 938 MDB_TOOL_IDL_FLUSH( be, txi ); 939 rc = mdb_txn_commit( txi ); 940 mdb_writes = 0; 941 for ( i=0; i<mi->mi_nattrs; i++ ) 942 mi->mi_attrs[i]->ai_cursor = NULL; 943 if( rc != 0 ) { 944 Debug( LDAP_DEBUG_ANY, 945 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) 946 ": txn_commit failed: %s (%d)\n", 947 mdb_strerror(rc), rc ); 948 e->e_id = NOID; 949 } 950 mdb_cursor_close( cursor ); 951 txi = NULL; 952 /* Must close the read txn to allow old pages to be reclaimed. */ 953 mdb_txn_abort( mdb_tool_txn ); 954 /* and then reopen it so that tool_entry_next still works. */ 955 mdb_txn_begin( mi->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn ); 956 mdb_cursor_open( mdb_tool_txn, mi->mi_id2entry, &cursor ); 957 key.mv_data = &id; 958 key.mv_size = sizeof(ID); 959 mdb_cursor_get( cursor, &key, NULL, MDB_SET ); 960 } 961 962 } else { 963 unsigned i; 964 mdb_writes = 0; 965 mdb_cursor_close( cursor ); 966 cursor = NULL; 967 mdb_txn_abort( txi ); 968 for ( i=0; i<mi->mi_nattrs; i++ ) 969 mi->mi_attrs[i]->ai_cursor = NULL; 970 Debug( LDAP_DEBUG_ANY, 971 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) 972 ": txn_aborted! err=%d\n", 973 rc ); 974 e->e_id = NOID; 975 txi = NULL; 976 } 977 mdb_entry_release( &op, e, 0 ); 978 979 return rc; 980 } 981 982 ID mdb_tool_entry_modify( 983 BackendDB *be, 984 Entry *e, 985 struct berval *text ) 986 { 987 int rc; 988 struct mdb_info *mdb; 989 Operation op = {0}; 990 Opheader ohdr = {0}; 991 992 assert( be != NULL ); 993 assert( slapMode & SLAP_TOOL_MODE ); 994 995 assert( text != NULL ); 996 assert( text->bv_val != NULL ); 997 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 998 999 assert ( e->e_id != NOID ); 1000 1001 Debug( LDAP_DEBUG_TRACE, 1002 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n", 1003 (long) e->e_id, e->e_dn ); 1004 1005 mdb = (struct mdb_info *) be->be_private; 1006 1007 if( cursor ) { 1008 mdb_cursor_close( cursor ); 1009 cursor = NULL; 1010 } 1011 if ( !mdb_tool_txn ) { 1012 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn ); 1013 if( rc != 0 ) { 1014 snprintf( text->bv_val, text->bv_len, 1015 "txn_begin failed: %s (%d)", 1016 mdb_strerror(rc), rc ); 1017 Debug( LDAP_DEBUG_ANY, 1018 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n", 1019 text->bv_val ); 1020 return NOID; 1021 } 1022 } 1023 1024 op.o_hdr = &ohdr; 1025 op.o_bd = be; 1026 op.o_tmpmemctx = NULL; 1027 op.o_tmpmfuncs = &ch_mfuncs; 1028 1029 /* id2entry index */ 1030 rc = mdb_id2entry_update( &op, mdb_tool_txn, NULL, e ); 1031 if( rc != 0 ) { 1032 snprintf( text->bv_val, text->bv_len, 1033 "id2entry_update failed: err=%d", rc ); 1034 Debug( LDAP_DEBUG_ANY, 1035 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n", 1036 text->bv_val ); 1037 goto done; 1038 } 1039 1040 done: 1041 if( rc == 0 ) { 1042 rc = mdb_txn_commit( mdb_tool_txn ); 1043 if( rc != 0 ) { 1044 mdb->mi_numads = 0; 1045 snprintf( text->bv_val, text->bv_len, 1046 "txn_commit failed: %s (%d)", 1047 mdb_strerror(rc), rc ); 1048 Debug( LDAP_DEBUG_ANY, 1049 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": " 1050 "%s\n", text->bv_val ); 1051 e->e_id = NOID; 1052 } 1053 1054 } else { 1055 mdb_txn_abort( mdb_tool_txn ); 1056 snprintf( text->bv_val, text->bv_len, 1057 "txn_aborted! %s (%d)", 1058 mdb_strerror(rc), rc ); 1059 Debug( LDAP_DEBUG_ANY, 1060 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n", 1061 text->bv_val ); 1062 e->e_id = NOID; 1063 } 1064 mdb_tool_txn = NULL; 1065 1066 return e->e_id; 1067 } 1068 1069 int mdb_tool_entry_delete( 1070 BackendDB *be, 1071 struct berval *ndn, 1072 struct berval *text ) 1073 { 1074 int rc; 1075 struct mdb_info *mdb; 1076 Operation op = {0}; 1077 Opheader ohdr = {0}; 1078 Entry *e; 1079 1080 assert( be != NULL ); 1081 assert( slapMode & SLAP_TOOL_MODE ); 1082 1083 assert( text != NULL ); 1084 assert( text->bv_val != NULL ); 1085 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 1086 1087 assert ( ndn != NULL ); 1088 assert ( ndn->bv_val != NULL ); 1089 1090 Debug( LDAP_DEBUG_TRACE, 1091 "=> " LDAP_XSTRING(mdb_tool_entry_delete) "( %s )\n", 1092 ndn->bv_val ); 1093 1094 mdb = (struct mdb_info *) be->be_private; 1095 1096 assert( cursor == NULL ); 1097 if( cursor ) { 1098 mdb_cursor_close( cursor ); 1099 cursor = NULL; 1100 } 1101 if( !mdb_tool_txn ) { 1102 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn ); 1103 if( rc != 0 ) { 1104 snprintf( text->bv_val, text->bv_len, 1105 "txn_begin failed: %s (%d)", 1106 mdb_strerror(rc), rc ); 1107 Debug( LDAP_DEBUG_ANY, 1108 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n", 1109 text->bv_val ); 1110 return LDAP_OTHER; 1111 } 1112 } 1113 1114 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &cursor ); 1115 if( rc != 0 ) { 1116 snprintf( text->bv_val, text->bv_len, 1117 "cursor_open failed: %s (%d)", 1118 mdb_strerror(rc), rc ); 1119 Debug( LDAP_DEBUG_ANY, 1120 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n", 1121 text->bv_val ); 1122 return LDAP_OTHER; 1123 } 1124 1125 op.o_hdr = &ohdr; 1126 op.o_bd = be; 1127 op.o_tmpmemctx = NULL; 1128 op.o_tmpmfuncs = &ch_mfuncs; 1129 1130 rc = mdb_dn2entry( &op, mdb_tool_txn, cursor, ndn, &e, NULL, 0 ); 1131 if( rc != 0 ) { 1132 snprintf( text->bv_val, text->bv_len, 1133 "dn2entry failed: %s (%d)", 1134 mdb_strerror(rc), rc ); 1135 Debug( LDAP_DEBUG_ANY, 1136 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n", 1137 text->bv_val ); 1138 goto done; 1139 } 1140 1141 /* check that we wouldn't orphan any children */ 1142 rc = mdb_dn2id_children( &op, mdb_tool_txn, e ); 1143 if( rc != MDB_NOTFOUND ) { 1144 switch( rc ) { 1145 case 0: 1146 snprintf( text->bv_val, text->bv_len, 1147 "delete failed:" 1148 " subordinate objects must be deleted first"); 1149 break; 1150 default: 1151 snprintf( text->bv_val, text->bv_len, 1152 "has_children failed: %s (%d)", 1153 mdb_strerror(rc), rc ); 1154 break; 1155 } 1156 rc = -1; 1157 Debug( LDAP_DEBUG_ANY, 1158 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n", 1159 text->bv_val ); 1160 goto done; 1161 } 1162 1163 /* delete from dn2id */ 1164 rc = mdb_dn2id_delete( &op, cursor, e->e_id, 1 ); 1165 if( rc != 0 ) { 1166 snprintf( text->bv_val, text->bv_len, 1167 "dn2id_delete failed: err=%d", rc ); 1168 Debug( LDAP_DEBUG_ANY, 1169 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n", 1170 text->bv_val ); 1171 goto done; 1172 } 1173 1174 /* deindex values */ 1175 rc = mdb_index_entry_del( &op, mdb_tool_txn, e ); 1176 if( rc != 0 ) { 1177 snprintf( text->bv_val, text->bv_len, 1178 "entry_delete failed: err=%d", rc ); 1179 Debug( LDAP_DEBUG_ANY, 1180 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n", 1181 text->bv_val ); 1182 goto done; 1183 } 1184 1185 /* do the deletion */ 1186 rc = mdb_id2entry_delete( be, mdb_tool_txn, e ); 1187 if( rc != 0 ) { 1188 snprintf( text->bv_val, text->bv_len, 1189 "id2entry_update failed: err=%d", rc ); 1190 Debug( LDAP_DEBUG_ANY, 1191 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n", 1192 text->bv_val ); 1193 goto done; 1194 } 1195 1196 done: 1197 /* free entry */ 1198 if( e != NULL ) { 1199 mdb_entry_return( &op, e ); 1200 } 1201 1202 if( rc == 0 ) { 1203 rc = mdb_txn_commit( mdb_tool_txn ); 1204 if( rc != 0 ) { 1205 snprintf( text->bv_val, text->bv_len, 1206 "txn_commit failed: %s (%d)", 1207 mdb_strerror(rc), rc ); 1208 Debug( LDAP_DEBUG_ANY, 1209 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": " 1210 "%s\n", text->bv_val ); 1211 } 1212 1213 } else { 1214 mdb_txn_abort( mdb_tool_txn ); 1215 snprintf( text->bv_val, text->bv_len, 1216 "txn_aborted! %s (%d)", 1217 mdb_strerror(rc), rc ); 1218 Debug( LDAP_DEBUG_ANY, 1219 "=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n", 1220 text->bv_val ); 1221 } 1222 mdb_tool_txn = NULL; 1223 cursor = NULL; 1224 1225 return rc; 1226 } 1227 1228 static void * 1229 mdb_tool_index_task( void *ctx, void *ptr ) 1230 { 1231 int base = *(int *)ptr; 1232 Operation op = {0}; 1233 Opheader ohdr = {0}; 1234 AttrIxInfo ai = {0}, *aio; 1235 1236 free( ptr ); 1237 op.o_hdr = &ohdr; 1238 op.o_bd = mdb_tool_ix_be; 1239 op.o_tmpmemctx = NULL; 1240 op.o_tmpmfuncs = &ch_mfuncs; 1241 aio = mdb_tool_axinfo[base]; 1242 mdb_tool_axinfo[base] = &ai; 1243 LDAP_SLIST_INSERT_HEAD( &op.o_extra, &ai.ai_oe, oe_next ); 1244 while ( 1 ) { 1245 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 1246 mdb_tool_index_tcount--; 1247 if ( !mdb_tool_index_tcount ) 1248 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main ); 1249 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work, 1250 &mdb_tool_index_mutex ); 1251 if ( slapd_shutdown ) { 1252 mdb_tool_index_tcount--; 1253 if ( !mdb_tool_index_tcount ) 1254 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main ); 1255 *aio = ai; 1256 mdb_tool_axinfo[base] = aio; 1257 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 1258 break; 1259 } 1260 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 1261 mdb_tool_index_rec[base].ir_i = mdb_index_recrun( &op, 1262 mdb_tool_ix_txn, 1263 mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base ); 1264 } 1265 1266 return NULL; 1267 } 1268 1269 #ifdef MDB_TOOL_IDL_CACHING 1270 static int 1271 mdb_tool_idl_cmp( const void *v1, const void *v2 ) 1272 { 1273 const mdb_tool_idl_cache *c1 = v1, *c2 = v2; 1274 int rc; 1275 1276 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc; 1277 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len ); 1278 } 1279 1280 static int 1281 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrIxInfo *ai, mdb_tool_idl_cache *ic ) 1282 { 1283 mdb_tool_idl_cache_entry *ice; 1284 MDB_val key, data[2]; 1285 int i, rc; 1286 ID id, nid; 1287 1288 /* Freshly allocated, ignore it */ 1289 if ( !ic->head && ic->count <= MDB_idl_db_size ) { 1290 return 0; 1291 } 1292 1293 key.mv_data = ic->kstr.bv_val; 1294 key.mv_size = ic->kstr.bv_len; 1295 1296 if ( ic->count > MDB_idl_db_size ) { 1297 while ( ic->flags & WAS_FOUND ) { 1298 rc = mdb_cursor_get( mc, &key, data, MDB_SET ); 1299 if ( rc ) { 1300 /* FIXME: find out why this happens */ 1301 ic->flags = 0; 1302 break; 1303 } 1304 if ( ic->flags & WAS_RANGE ) { 1305 /* Skip lo */ 1306 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); 1307 1308 /* Get hi */ 1309 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); 1310 1311 /* Store range hi */ 1312 data[0].mv_data = &ic->last; 1313 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT ); 1314 } else { 1315 /* Delete old data, replace with range */ 1316 ic->first = *(ID *)data[0].mv_data; 1317 mdb_cursor_del( mc, MDB_NODUPDATA ); 1318 } 1319 break; 1320 } 1321 if ( !(ic->flags & WAS_RANGE)) { 1322 /* range, didn't exist before */ 1323 nid = 0; 1324 data[0].mv_size = sizeof(ID); 1325 data[0].mv_data = &nid; 1326 rc = mdb_cursor_put( mc, &key, data, 0 ); 1327 if ( rc == 0 ) { 1328 data[0].mv_data = &ic->first; 1329 rc = mdb_cursor_put( mc, &key, data, 0 ); 1330 if ( rc == 0 ) { 1331 data[0].mv_data = &ic->last; 1332 rc = mdb_cursor_put( mc, &key, data, 0 ); 1333 } 1334 } 1335 if ( rc ) { 1336 rc = -1; 1337 } 1338 } 1339 } else { 1340 /* Normal write */ 1341 int n; 1342 1343 data[0].mv_size = sizeof(ID); 1344 rc = 0; 1345 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) { 1346 int end; 1347 if ( ice->next ) { 1348 end = IDBLOCK; 1349 } else { 1350 end = (ic->count-ic->offset) & (IDBLOCK-1); 1351 if ( !end ) 1352 end = IDBLOCK; 1353 } 1354 data[1].mv_size = end; 1355 data[0].mv_data = ice->ids; 1356 rc = mdb_cursor_put( mc, &key, data, MDB_APPENDDUP|MDB_MULTIPLE ); 1357 if ( rc ) { 1358 rc = -1; 1359 break; 1360 } 1361 } 1362 if ( ic->head ) { 1363 ic->tail->next = ai->ai_flist; 1364 ai->ai_flist = ic->head; 1365 } 1366 } 1367 ic->head = ai->ai_clist; 1368 ai->ai_clist = ic; 1369 return rc; 1370 } 1371 1372 static int 1373 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai, AttrIxInfo *ax ) 1374 { 1375 MDB_cursor *mc; 1376 Avlnode *root; 1377 int rc; 1378 1379 mdb_cursor_open( txn, ai->ai_dbi, &mc ); 1380 root = ldap_tavl_end( ai->ai_root, TAVL_DIR_LEFT ); 1381 do { 1382 rc = mdb_tool_idl_flush_one( mc, ax, root->avl_data ); 1383 if ( rc != -1 ) 1384 rc = 0; 1385 } while ((root = ldap_tavl_next(root, TAVL_DIR_RIGHT))); 1386 mdb_cursor_close( mc ); 1387 1388 return rc; 1389 } 1390 1391 static int 1392 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn ) 1393 { 1394 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 1395 int rc = 0; 1396 unsigned int i, dbi; 1397 1398 for ( i=0; i < mdb->mi_nattrs; i++ ) { 1399 if ( !mdb->mi_attrs[i]->ai_root ) continue; 1400 rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i], mdb_tool_axinfo[i % mdb_tool_threads] ); 1401 ldap_tavl_free(mdb->mi_attrs[i]->ai_root, NULL); 1402 mdb->mi_attrs[i]->ai_root = NULL; 1403 if ( rc ) 1404 break; 1405 } 1406 return rc; 1407 } 1408 1409 int mdb_tool_idl_add( 1410 BackendDB *be, 1411 MDB_cursor *mc, 1412 struct berval *keys, 1413 ID id ) 1414 { 1415 MDB_dbi dbi; 1416 mdb_tool_idl_cache *ic, itmp; 1417 mdb_tool_idl_cache_entry *ice; 1418 int i, rc, lcount; 1419 AttrIxInfo *ax = (AttrIxInfo *)mc; 1420 AttrInfo *ai = (AttrInfo *)ax->ai_ai; 1421 mc = ai->ai_cursor; 1422 1423 dbi = ai->ai_dbi; 1424 for (i=0; keys[i].bv_val; i++) { 1425 itmp.kstr = keys[i]; 1426 ic = ldap_tavl_find( ai->ai_root, &itmp, mdb_tool_idl_cmp ); 1427 1428 /* No entry yet, create one */ 1429 if ( !ic ) { 1430 MDB_val key, data; 1431 ID nid; 1432 int rc; 1433 1434 if ( ax->ai_clist ) { 1435 ic = ax->ai_clist; 1436 ax->ai_clist = ic->head; 1437 } else { 1438 ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 ); 1439 } 1440 ic->kstr.bv_len = itmp.kstr.bv_len; 1441 ic->kstr.bv_val = (char *)(ic+1); 1442 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len ); 1443 ic->head = ic->tail = NULL; 1444 ic->last = 0; 1445 ic->count = 0; 1446 ic->offset = 0; 1447 ic->flags = 0; 1448 ldap_tavl_insert( &ai->ai_root, ic, mdb_tool_idl_cmp, 1449 ldap_avl_dup_error ); 1450 1451 /* load existing key count here */ 1452 key.mv_size = keys[i].bv_len; 1453 key.mv_data = keys[i].bv_val; 1454 rc = mdb_cursor_get( mc, &key, &data, MDB_SET ); 1455 if ( rc == 0 ) { 1456 ic->flags |= WAS_FOUND; 1457 nid = *(ID *)data.mv_data; 1458 if ( nid == 0 ) { 1459 ic->count = MDB_idl_db_size+1; 1460 ic->flags |= WAS_RANGE; 1461 } else { 1462 size_t count; 1463 1464 mdb_cursor_count( mc, &count ); 1465 ic->count = count; 1466 ic->first = nid; 1467 ic->offset = count & (IDBLOCK-1); 1468 } 1469 } 1470 } 1471 /* are we a range already? */ 1472 if ( ic->count > MDB_idl_db_size ) { 1473 ic->last = id; 1474 continue; 1475 /* Are we at the limit, and converting to a range? */ 1476 } else if ( ic->count == MDB_idl_db_size ) { 1477 if ( ic->head ) { 1478 ic->tail->next = ax->ai_flist; 1479 ax->ai_flist = ic->head; 1480 } 1481 ic->head = ic->tail = NULL; 1482 ic->last = id; 1483 ic->count++; 1484 continue; 1485 } 1486 /* No free block, create that too */ 1487 lcount = (ic->count-ic->offset) & (IDBLOCK-1); 1488 if ( !ic->tail || lcount == 0) { 1489 if ( ax->ai_flist ) { 1490 ice = ax->ai_flist; 1491 ax->ai_flist = ice->next; 1492 } else { 1493 ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry )); 1494 } 1495 ice->next = NULL; 1496 if ( !ic->head ) { 1497 ic->head = ice; 1498 } else { 1499 ic->tail->next = ice; 1500 } 1501 ic->tail = ice; 1502 if ( lcount ) 1503 ice->ids[lcount-1] = 0; 1504 if ( !ic->count ) 1505 ic->first = id; 1506 } 1507 ice = ic->tail; 1508 if (!lcount || ice->ids[lcount-1] != id) { 1509 ice->ids[lcount] = id; 1510 ic->count++; 1511 } 1512 } 1513 1514 return 0; 1515 } 1516 #endif /* MDB_TOOL_IDL_CACHING */ 1517 1518 /* Upgrade from pre 2.4.34 dn2id format */ 1519 1520 #include <ac/unistd.h> 1521 #include <lutil_meter.h> 1522 1523 #define STACKSIZ 2048 1524 1525 typedef struct rec { 1526 ID id; 1527 size_t len; 1528 char rdn[512]; 1529 } rec; 1530 1531 static int 1532 mdb_dn2id_upgrade( BackendDB *be ) { 1533 struct mdb_info *mi = (struct mdb_info *) be->be_private; 1534 MDB_txn *mt; 1535 MDB_cursor *mc = NULL; 1536 MDB_val key, data; 1537 int rc, writes=0, depth=0; 1538 int enable_meter = 0; 1539 ID id = 0, *num, count = 0; 1540 rec *stack; 1541 lutil_meter_t meter; 1542 1543 if (!(mi->mi_flags & MDB_NEED_UPGRADE)) { 1544 Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n", 1545 be->be_suffix[0].bv_val ); 1546 return 0; 1547 } 1548 1549 { 1550 MDB_stat st; 1551 1552 mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st); 1553 if (!st.ms_entries) { 1554 /* Empty DB, nothing to upgrade? */ 1555 return 0; 1556 } 1557 if (isatty(2)) 1558 enable_meter = !lutil_meter_open(&meter, 1559 &lutil_meter_text_display, 1560 &lutil_meter_linear_estimator, 1561 st.ms_entries); 1562 } 1563 1564 num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec))); 1565 stack = (rec *)(num + STACKSIZ); 1566 1567 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); 1568 if (rc) { 1569 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n", 1570 mdb_strerror(rc), rc ); 1571 goto leave; 1572 } 1573 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); 1574 if (rc) { 1575 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n", 1576 mdb_strerror(rc), rc ); 1577 goto leave; 1578 } 1579 1580 key.mv_size = sizeof(ID); 1581 /* post-order depth-first update */ 1582 for(;;) { 1583 size_t dkids; 1584 unsigned char *ptr; 1585 1586 /* visit */ 1587 key.mv_data = &id; 1588 stack[depth].id = id; 1589 rc = mdb_cursor_get(mc, &key, &data, MDB_SET); 1590 if (rc) { 1591 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n", 1592 mdb_strerror(rc), rc ); 1593 goto leave; 1594 } 1595 num[depth] = 1; 1596 1597 rc = mdb_cursor_count(mc, &dkids); 1598 if (rc) { 1599 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n", 1600 mdb_strerror(rc), rc ); 1601 goto leave; 1602 } 1603 if (dkids > 1) { 1604 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); 1605 down: 1606 ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID); 1607 memcpy(&id, ptr, sizeof(ID)); 1608 depth++; 1609 memcpy(stack[depth].rdn, data.mv_data, data.mv_size); 1610 stack[depth].len = data.mv_size; 1611 continue; 1612 } 1613 1614 1615 /* pop: write updated count, advance to next node */ 1616 pop: 1617 /* update superior counts */ 1618 if (depth) 1619 num[depth-1] += num[depth]; 1620 1621 key.mv_data = &id; 1622 id = stack[depth-1].id; 1623 data.mv_data = stack[depth].rdn; 1624 data.mv_size = stack[depth].len; 1625 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); 1626 if (rc) { 1627 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n", 1628 mdb_strerror(rc), rc ); 1629 goto leave; 1630 } 1631 data.mv_data = stack[depth].rdn; 1632 ptr = (unsigned char *)data.mv_data + data.mv_size; 1633 memcpy(ptr, &num[depth], sizeof(ID)); 1634 data.mv_size += sizeof(ID); 1635 rc = mdb_cursor_del(mc, 0); 1636 if (rc) { 1637 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n", 1638 mdb_strerror(rc), rc ); 1639 goto leave; 1640 } 1641 rc = mdb_cursor_put(mc, &key, &data, 0); 1642 if (rc) { 1643 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n", 1644 mdb_strerror(rc), rc ); 1645 goto leave; 1646 } 1647 count++; 1648 #if 1 1649 if (enable_meter) 1650 lutil_meter_update(&meter, count, 0); 1651 #else 1652 { 1653 int len; 1654 ptr = data.mv_data; 1655 len = (ptr[0] & 0x7f) << 8 | ptr[1]; 1656 printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2); 1657 } 1658 #endif 1659 writes++; 1660 if (writes == 1000) { 1661 mdb_cursor_close(mc); 1662 rc = mdb_txn_commit(mt); 1663 if (rc) { 1664 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n", 1665 mdb_strerror(rc), rc ); 1666 goto leave; 1667 } 1668 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); 1669 if (rc) { 1670 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n", 1671 mdb_strerror(rc), rc ); 1672 goto leave; 1673 } 1674 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); 1675 if (rc) { 1676 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n", 1677 mdb_strerror(rc), rc ); 1678 goto leave; 1679 } 1680 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); 1681 if (rc) { 1682 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n", 1683 mdb_strerror(rc), rc ); 1684 goto leave; 1685 } 1686 writes = 0; 1687 } 1688 depth--; 1689 1690 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); 1691 if (rc == 0) 1692 goto down; 1693 rc = 0; 1694 if (depth) 1695 goto pop; 1696 else 1697 break; 1698 } 1699 leave: 1700 mdb_cursor_close(mc); 1701 if (mt) { 1702 int r2; 1703 r2 = mdb_txn_commit(mt); 1704 if (r2) { 1705 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n", 1706 mdb_strerror(r2), r2 ); 1707 if (!rc) 1708 rc = r2; 1709 } 1710 } 1711 ch_free(num); 1712 if (enable_meter) { 1713 lutil_meter_update(&meter, count, 1); 1714 lutil_meter_close(&meter); 1715 } 1716 return rc; 1717 } 1718