1 /* $NetBSD: tools.c,v 1.1.1.1 2014/05/28 09:58:50 tron Exp $ */ 2 3 /* tools.c - tools for slap tools */ 4 /* $OpenLDAP$ */ 5 /* This work is part of OpenLDAP Software <http://www.openldap.org/>. 6 * 7 * Copyright 2011-2014 The OpenLDAP Foundation. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted only as authorized by the OpenLDAP 12 * Public License. 13 * 14 * A copy of this license is available in the file LICENSE in the 15 * top-level directory of the distribution or, alternatively, at 16 * <http://www.OpenLDAP.org/license.html>. 17 */ 18 19 #include "portable.h" 20 21 #include <stdio.h> 22 #include <ac/string.h> 23 #include <ac/errno.h> 24 25 #define AVL_INTERNAL 26 #include "back-mdb.h" 27 #include "idl.h" 28 29 #ifdef MDB_TOOL_IDL_CACHING 30 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn ); 31 32 #define IDBLOCK 1024 33 34 typedef struct mdb_tool_idl_cache_entry { 35 struct mdb_tool_idl_cache_entry *next; 36 ID ids[IDBLOCK]; 37 } mdb_tool_idl_cache_entry; 38 39 typedef struct mdb_tool_idl_cache { 40 struct berval kstr; 41 mdb_tool_idl_cache_entry *head, *tail; 42 ID first, last; 43 int count; 44 short offset; 45 short flags; 46 } mdb_tool_idl_cache; 47 #define WAS_FOUND 0x01 48 #define WAS_RANGE 0x02 49 50 #define MDB_TOOL_IDL_FLUSH(be, txn) mdb_tool_idl_flush(be, txn) 51 #else 52 #define MDB_TOOL_IDL_FLUSH(be, txn) 53 #endif /* MDB_TOOL_IDL_CACHING */ 54 55 static MDB_txn *txn = NULL, *txi = NULL; 56 static MDB_cursor *cursor = NULL, *idcursor = NULL; 57 static MDB_cursor *mcp = NULL, *mcd = NULL; 58 static MDB_val key, data; 59 static ID previd = NOID; 60 61 typedef struct dn_id { 62 ID id; 63 struct berval dn; 64 } dn_id; 65 66 #define HOLE_SIZE 4096 67 static dn_id hbuf[HOLE_SIZE], *holes = hbuf; 68 static unsigned nhmax = HOLE_SIZE; 69 static unsigned nholes; 70 71 static struct berval *tool_base; 72 static int tool_scope; 73 static Filter *tool_filter; 74 static Entry *tool_next_entry; 75 76 static ID mdb_tool_ix_id; 77 static Operation *mdb_tool_ix_op; 78 static MDB_txn *mdb_tool_ix_txn; 79 static int mdb_tool_index_tcount, mdb_tool_threads; 80 static IndexRec *mdb_tool_index_rec; 81 static struct mdb_info *mdb_tool_info; 82 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex; 83 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main; 84 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work; 85 static void * mdb_tool_index_task( void *ctx, void *ptr ); 86 87 static int mdb_writes, mdb_writes_per_commit; 88 89 /* Number of ops per commit in Quick mode. 90 * Batching speeds writes overall, but too large a 91 * batch will fail with MDB_TXN_FULL. 92 */ 93 #ifndef MDB_WRITES_PER_COMMIT 94 #define MDB_WRITES_PER_COMMIT 500 95 #endif 96 97 static int 98 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ); 99 100 int mdb_tool_entry_open( 101 BackendDB *be, int mode ) 102 { 103 /* In Quick mode, commit once per 500 entries */ 104 mdb_writes = 0; 105 if ( slapMode & SLAP_TOOL_QUICK ) 106 mdb_writes_per_commit = MDB_WRITES_PER_COMMIT; 107 else 108 mdb_writes_per_commit = 1; 109 110 /* Set up for threaded slapindex */ 111 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) { 112 if ( !mdb_tool_info ) { 113 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 114 ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex ); 115 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main ); 116 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work ); 117 if ( mdb->mi_nattrs ) { 118 int i; 119 #if 0 /* threaded indexing has no performance advantage */ 120 mdb_tool_threads = slap_tool_thread_max - 1; 121 #endif 122 if ( mdb_tool_threads > 1 ) { 123 mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec )); 124 mdb_tool_index_tcount = mdb_tool_threads - 1; 125 for (i=1; i<mdb_tool_threads; i++) { 126 int *ptr = ch_malloc( sizeof( int )); 127 *ptr = i; 128 ldap_pvt_thread_pool_submit( &connection_pool, 129 mdb_tool_index_task, ptr ); 130 } 131 mdb_tool_info = mdb; 132 } 133 } 134 } 135 } 136 137 return 0; 138 } 139 140 int mdb_tool_entry_close( 141 BackendDB *be ) 142 { 143 if ( mdb_tool_info ) { 144 slapd_shutdown = 1; 145 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 146 147 /* There might still be some threads starting */ 148 while ( mdb_tool_index_tcount > 0 ) { 149 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 150 &mdb_tool_index_mutex ); 151 } 152 153 mdb_tool_index_tcount = mdb_tool_threads - 1; 154 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work ); 155 156 /* Make sure all threads are stopped */ 157 while ( mdb_tool_index_tcount > 0 ) { 158 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 159 &mdb_tool_index_mutex ); 160 } 161 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 162 163 mdb_tool_info = NULL; 164 slapd_shutdown = 0; 165 ch_free( mdb_tool_index_rec ); 166 mdb_tool_index_tcount = mdb_tool_threads - 1; 167 } 168 169 if( idcursor ) { 170 mdb_cursor_close( idcursor ); 171 idcursor = NULL; 172 } 173 if( cursor ) { 174 mdb_cursor_close( cursor ); 175 cursor = NULL; 176 } 177 if( txn ) { 178 int rc; 179 MDB_TOOL_IDL_FLUSH( be, txn ); 180 if (( rc = mdb_txn_commit( txn ))) { 181 Debug( LDAP_DEBUG_ANY, 182 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: " 183 "txn_commit failed: %s (%d)\n", 184 be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); 185 return -1; 186 } 187 txn = NULL; 188 } 189 190 if( nholes ) { 191 unsigned i; 192 fprintf( stderr, "Error, entries missing!\n"); 193 for (i=0; i<nholes; i++) { 194 fprintf(stderr, " entry %ld: %s\n", 195 holes[i].id, holes[i].dn.bv_val); 196 } 197 nholes = 0; 198 return -1; 199 } 200 201 return 0; 202 } 203 204 ID 205 mdb_tool_entry_first_x( 206 BackendDB *be, 207 struct berval *base, 208 int scope, 209 Filter *f ) 210 { 211 tool_base = base; 212 tool_scope = scope; 213 tool_filter = f; 214 215 return mdb_tool_entry_next( be ); 216 } 217 218 ID mdb_tool_entry_next( 219 BackendDB *be ) 220 { 221 int rc; 222 ID id; 223 struct mdb_info *mdb; 224 225 assert( be != NULL ); 226 assert( slapMode & SLAP_TOOL_MODE ); 227 228 mdb = (struct mdb_info *) be->be_private; 229 assert( mdb != NULL ); 230 231 if ( !txn ) { 232 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &txn ); 233 if ( rc ) 234 return NOID; 235 rc = mdb_cursor_open( txn, mdb->mi_id2entry, &cursor ); 236 if ( rc ) { 237 mdb_txn_abort( txn ); 238 return NOID; 239 } 240 } 241 242 next:; 243 rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT ); 244 245 if( rc ) { 246 return NOID; 247 } 248 249 previd = *(ID *)key.mv_data; 250 id = previd; 251 252 if ( !data.mv_size ) 253 goto next; 254 255 if ( tool_filter || tool_base ) { 256 static Operation op = {0}; 257 static Opheader ohdr = {0}; 258 259 op.o_hdr = &ohdr; 260 op.o_bd = be; 261 op.o_tmpmemctx = NULL; 262 op.o_tmpmfuncs = &ch_mfuncs; 263 264 if ( tool_next_entry ) { 265 mdb_entry_release( &op, tool_next_entry, 0 ); 266 tool_next_entry = NULL; 267 } 268 269 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry ); 270 if ( rc == LDAP_NO_SUCH_OBJECT ) { 271 goto next; 272 } 273 274 assert( tool_next_entry != NULL ); 275 276 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE ) 277 { 278 mdb_entry_release( &op, tool_next_entry, 0 ); 279 tool_next_entry = NULL; 280 goto next; 281 } 282 } 283 284 return id; 285 } 286 287 ID mdb_tool_dn2id_get( 288 Backend *be, 289 struct berval *dn 290 ) 291 { 292 struct mdb_info *mdb; 293 Operation op = {0}; 294 Opheader ohdr = {0}; 295 ID id; 296 int rc; 297 298 if ( BER_BVISEMPTY(dn) ) 299 return 0; 300 301 mdb = (struct mdb_info *) be->be_private; 302 303 if ( !txn ) { 304 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ? 305 MDB_RDONLY : 0, &txn ); 306 if ( rc ) 307 return NOID; 308 } 309 310 op.o_hdr = &ohdr; 311 op.o_bd = be; 312 op.o_tmpmemctx = NULL; 313 op.o_tmpmfuncs = &ch_mfuncs; 314 315 rc = mdb_dn2id( &op, txn, NULL, dn, &id, NULL, NULL, NULL ); 316 if ( rc == MDB_NOTFOUND ) 317 return NOID; 318 319 return id; 320 } 321 322 static int 323 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ) 324 { 325 Operation op = {0}; 326 Opheader ohdr = {0}; 327 328 Entry *e = NULL; 329 struct berval dn = BER_BVNULL, ndn = BER_BVNULL; 330 int rc; 331 332 assert( be != NULL ); 333 assert( slapMode & SLAP_TOOL_MODE ); 334 335 if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) { 336 *ep = tool_next_entry; 337 tool_next_entry = NULL; 338 return LDAP_SUCCESS; 339 } 340 341 if ( id != previd ) { 342 key.mv_size = sizeof(ID); 343 key.mv_data = &id; 344 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET ); 345 if ( rc ) { 346 rc = LDAP_OTHER; 347 goto done; 348 } 349 } 350 if ( !data.mv_size ) { 351 rc = LDAP_NO_SUCH_OBJECT; 352 goto done; 353 } 354 355 op.o_hdr = &ohdr; 356 op.o_bd = be; 357 op.o_tmpmemctx = NULL; 358 op.o_tmpmfuncs = &ch_mfuncs; 359 if ( slapMode & SLAP_TOOL_READONLY ) { 360 rc = mdb_id2name( &op, txn, &idcursor, id, &dn, &ndn ); 361 if ( rc ) { 362 rc = LDAP_OTHER; 363 if ( e ) { 364 mdb_entry_return( &op, e ); 365 e = NULL; 366 } 367 goto done; 368 } 369 if ( tool_base != NULL ) { 370 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) { 371 ch_free( dn.bv_val ); 372 ch_free( ndn.bv_val ); 373 rc = LDAP_NO_SUCH_OBJECT; 374 goto done; 375 } 376 } 377 } 378 rc = mdb_entry_decode( &op, &data, &e ); 379 e->e_id = id; 380 if ( !BER_BVISNULL( &dn )) { 381 e->e_name = dn; 382 e->e_nname = ndn; 383 } else { 384 e->e_name.bv_val = NULL; 385 e->e_nname.bv_val = NULL; 386 } 387 388 done: 389 if ( e != NULL ) { 390 *ep = e; 391 } 392 393 return rc; 394 } 395 396 Entry* 397 mdb_tool_entry_get( BackendDB *be, ID id ) 398 { 399 Entry *e = NULL; 400 int rc; 401 402 if ( !txn ) { 403 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 404 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 405 (slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &txn ); 406 if ( rc ) 407 return NULL; 408 } 409 if ( !cursor ) { 410 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 411 rc = mdb_cursor_open( txn, mdb->mi_id2entry, &cursor ); 412 if ( rc ) { 413 mdb_txn_abort( txn ); 414 txn = NULL; 415 return NULL; 416 } 417 } 418 (void)mdb_tool_entry_get_int( be, id, &e ); 419 return e; 420 } 421 422 static int mdb_tool_next_id( 423 Operation *op, 424 MDB_txn *tid, 425 Entry *e, 426 struct berval *text, 427 int hole ) 428 { 429 struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; 430 struct berval dn = e->e_name; 431 struct berval ndn = e->e_nname; 432 struct berval pdn, npdn, nmatched; 433 ID id, pid = 0; 434 int rc; 435 436 if (ndn.bv_len == 0) { 437 e->e_id = 0; 438 return 0; 439 } 440 441 rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, NULL, &nmatched ); 442 if ( rc == MDB_NOTFOUND ) { 443 if ( !be_issuffix( op->o_bd, &ndn ) ) { 444 ID eid = e->e_id; 445 dnParent( &ndn, &npdn ); 446 if ( nmatched.bv_len != npdn.bv_len ) { 447 dnParent( &dn, &pdn ); 448 e->e_name = pdn; 449 e->e_nname = npdn; 450 rc = mdb_tool_next_id( op, tid, e, text, 1 ); 451 e->e_name = dn; 452 e->e_nname = ndn; 453 if ( rc ) { 454 return rc; 455 } 456 /* If parent didn't exist, it was created just now 457 * and its ID is now in e->e_id. Make sure the current 458 * entry gets added under the new parent ID. 459 */ 460 if ( eid != e->e_id ) { 461 pid = e->e_id; 462 } 463 } else { 464 pid = id; 465 } 466 } 467 rc = mdb_next_id( op->o_bd, idcursor, &e->e_id ); 468 if ( rc ) { 469 snprintf( text->bv_val, text->bv_len, 470 "next_id failed: %s (%d)", 471 mdb_strerror(rc), rc ); 472 Debug( LDAP_DEBUG_ANY, 473 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 474 return rc; 475 } 476 rc = mdb_dn2id_add( op, mcp, mcd, pid, 1, 1, e ); 477 if ( rc ) { 478 snprintf( text->bv_val, text->bv_len, 479 "dn2id_add failed: %s (%d)", 480 mdb_strerror(rc), rc ); 481 Debug( LDAP_DEBUG_ANY, 482 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 483 } else if ( hole ) { 484 MDB_val key, data; 485 if ( nholes == nhmax - 1 ) { 486 if ( holes == hbuf ) { 487 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 ); 488 AC_MEMCPY( holes, hbuf, sizeof(hbuf) ); 489 } else { 490 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 ); 491 } 492 nhmax *= 2; 493 } 494 ber_dupbv( &holes[nholes].dn, &ndn ); 495 holes[nholes++].id = e->e_id; 496 key.mv_size = sizeof(ID); 497 key.mv_data = &e->e_id; 498 data.mv_size = 0; 499 data.mv_data = NULL; 500 rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE ); 501 if ( rc == MDB_KEYEXIST ) 502 rc = 0; 503 if ( rc ) { 504 snprintf( text->bv_val, text->bv_len, 505 "dummy id2entry add failed: %s (%d)", 506 mdb_strerror(rc), rc ); 507 Debug( LDAP_DEBUG_ANY, 508 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 509 } 510 } 511 } else if ( !hole ) { 512 unsigned i, j; 513 514 e->e_id = id; 515 516 for ( i=0; i<nholes; i++) { 517 if ( holes[i].id == e->e_id ) { 518 free(holes[i].dn.bv_val); 519 for (j=i;j<nholes;j++) holes[j] = holes[j+1]; 520 holes[j].id = 0; 521 nholes--; 522 break; 523 } else if ( holes[i].id > e->e_id ) { 524 break; 525 } 526 } 527 } 528 return rc; 529 } 530 531 static int 532 mdb_tool_index_add( 533 Operation *op, 534 MDB_txn *txn, 535 Entry *e ) 536 { 537 struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; 538 539 if ( !mdb->mi_nattrs ) 540 return 0; 541 542 if ( mdb_tool_threads > 1 ) { 543 IndexRec *ir; 544 int i, rc; 545 Attribute *a; 546 547 ir = mdb_tool_index_rec; 548 for (i=0; i<mdb->mi_nattrs; i++) 549 ir[i].ir_attrs = NULL; 550 551 for ( a = e->e_attrs; a != NULL; a = a->a_next ) { 552 rc = mdb_index_recset( mdb, a, a->a_desc->ad_type, 553 &a->a_desc->ad_tags, ir ); 554 if ( rc ) 555 return rc; 556 } 557 for (i=0; i<mdb->mi_nattrs; i++) { 558 if ( !ir[i].ir_ai ) 559 break; 560 rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi, 561 &ir[i].ir_ai->ai_cursor ); 562 if ( rc ) 563 return rc; 564 } 565 mdb_tool_ix_id = e->e_id; 566 mdb_tool_ix_op = op; 567 mdb_tool_ix_txn = txn; 568 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 569 /* Wait for all threads to be ready */ 570 while ( mdb_tool_index_tcount ) { 571 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 572 &mdb_tool_index_mutex ); 573 } 574 575 for ( i=1; i<mdb_tool_threads; i++ ) 576 mdb_tool_index_rec[i].ir_i = LDAP_BUSY; 577 mdb_tool_index_tcount = mdb_tool_threads - 1; 578 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 579 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work ); 580 581 rc = mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 ); 582 if ( rc ) 583 return rc; 584 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 585 for ( i=1; i<mdb_tool_threads; i++ ) { 586 if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) { 587 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 588 &mdb_tool_index_mutex ); 589 i--; 590 continue; 591 } 592 if ( mdb_tool_index_rec[i].ir_i ) { 593 rc = mdb_tool_index_rec[i].ir_i; 594 break; 595 } 596 } 597 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 598 return rc; 599 } else 600 { 601 return mdb_index_entry_add( op, txn, e ); 602 } 603 } 604 605 ID mdb_tool_entry_put( 606 BackendDB *be, 607 Entry *e, 608 struct berval *text ) 609 { 610 int rc; 611 struct mdb_info *mdb; 612 Operation op = {0}; 613 Opheader ohdr = {0}; 614 615 assert( be != NULL ); 616 assert( slapMode & SLAP_TOOL_MODE ); 617 618 assert( text != NULL ); 619 assert( text->bv_val != NULL ); 620 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 621 622 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put) 623 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 ); 624 625 mdb = (struct mdb_info *) be->be_private; 626 627 if ( !txn ) { 628 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn ); 629 if( rc != 0 ) { 630 snprintf( text->bv_val, text->bv_len, 631 "txn_begin failed: %s (%d)", 632 mdb_strerror(rc), rc ); 633 Debug( LDAP_DEBUG_ANY, 634 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 635 text->bv_val, 0, 0 ); 636 return NOID; 637 } 638 rc = mdb_cursor_open( txn, mdb->mi_id2entry, &idcursor ); 639 if( rc != 0 ) { 640 snprintf( text->bv_val, text->bv_len, 641 "cursor_open failed: %s (%d)", 642 mdb_strerror(rc), rc ); 643 Debug( LDAP_DEBUG_ANY, 644 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 645 text->bv_val, 0, 0 ); 646 return NOID; 647 } 648 if ( !mdb->mi_nextid ) { 649 ID dummy; 650 mdb_next_id( be, idcursor, &dummy ); 651 } 652 rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcp ); 653 if( rc != 0 ) { 654 snprintf( text->bv_val, text->bv_len, 655 "cursor_open failed: %s (%d)", 656 mdb_strerror(rc), rc ); 657 Debug( LDAP_DEBUG_ANY, 658 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 659 text->bv_val, 0, 0 ); 660 return NOID; 661 } 662 rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcd ); 663 if( rc != 0 ) { 664 snprintf( text->bv_val, text->bv_len, 665 "cursor_open failed: %s (%d)", 666 mdb_strerror(rc), rc ); 667 Debug( LDAP_DEBUG_ANY, 668 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 669 text->bv_val, 0, 0 ); 670 return NOID; 671 } 672 } 673 674 op.o_hdr = &ohdr; 675 op.o_bd = be; 676 op.o_tmpmemctx = NULL; 677 op.o_tmpmfuncs = &ch_mfuncs; 678 679 /* add dn2id indices */ 680 rc = mdb_tool_next_id( &op, txn, e, text, 0 ); 681 if( rc != 0 ) { 682 goto done; 683 } 684 685 rc = mdb_tool_index_add( &op, txn, e ); 686 if( rc != 0 ) { 687 snprintf( text->bv_val, text->bv_len, 688 "index_entry_add failed: err=%d", rc ); 689 Debug( LDAP_DEBUG_ANY, 690 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 691 text->bv_val, 0, 0 ); 692 goto done; 693 } 694 695 696 /* id2entry index */ 697 rc = mdb_id2entry_add( &op, txn, idcursor, e ); 698 if( rc != 0 ) { 699 snprintf( text->bv_val, text->bv_len, 700 "id2entry_add failed: err=%d", rc ); 701 Debug( LDAP_DEBUG_ANY, 702 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 703 text->bv_val, 0, 0 ); 704 goto done; 705 } 706 707 done: 708 if( rc == 0 ) { 709 mdb_writes++; 710 if ( mdb_writes >= mdb_writes_per_commit ) { 711 unsigned i; 712 MDB_TOOL_IDL_FLUSH( be, txn ); 713 rc = mdb_txn_commit( txn ); 714 for ( i=0; i<mdb->mi_nattrs; i++ ) 715 mdb->mi_attrs[i]->ai_cursor = NULL; 716 mdb_writes = 0; 717 txn = NULL; 718 idcursor = NULL; 719 if( rc != 0 ) { 720 snprintf( text->bv_val, text->bv_len, 721 "txn_commit failed: %s (%d)", 722 mdb_strerror(rc), rc ); 723 Debug( LDAP_DEBUG_ANY, 724 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 725 text->bv_val, 0, 0 ); 726 e->e_id = NOID; 727 } 728 } 729 730 } else { 731 unsigned i; 732 mdb_txn_abort( txn ); 733 txn = NULL; 734 idcursor = NULL; 735 for ( i=0; i<mdb->mi_nattrs; i++ ) 736 mdb->mi_attrs[i]->ai_cursor = NULL; 737 mdb_writes = 0; 738 snprintf( text->bv_val, text->bv_len, 739 "txn_aborted! %s (%d)", 740 rc == LDAP_OTHER ? "Internal error" : 741 mdb_strerror(rc), rc ); 742 Debug( LDAP_DEBUG_ANY, 743 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 744 text->bv_val, 0, 0 ); 745 e->e_id = NOID; 746 } 747 748 return e->e_id; 749 } 750 751 static int mdb_dn2id_upgrade( BackendDB *be ); 752 753 int mdb_tool_entry_reindex( 754 BackendDB *be, 755 ID id, 756 AttributeDescription **adv ) 757 { 758 struct mdb_info *mi = (struct mdb_info *) be->be_private; 759 int rc; 760 Entry *e; 761 Operation op = {0}; 762 Opheader ohdr = {0}; 763 764 Debug( LDAP_DEBUG_ARGS, 765 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n", 766 (long) id, 0, 0 ); 767 assert( tool_base == NULL ); 768 assert( tool_filter == NULL ); 769 770 /* Special: do a dn2id upgrade */ 771 if ( adv && adv[0] == slap_schema.si_ad_entryDN ) { 772 /* short-circuit tool_entry_next() */ 773 mdb_cursor_get( cursor, &key, &data, MDB_LAST ); 774 return mdb_dn2id_upgrade( be ); 775 } 776 777 /* No indexes configured, nothing to do. Could return an 778 * error here to shortcut things. 779 */ 780 if (!mi->mi_attrs) { 781 return 0; 782 } 783 784 /* Check for explicit list of attrs to index */ 785 if ( adv ) { 786 int i, j, n; 787 788 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) { 789 /* count */ 790 for ( n = 0; adv[n]; n++ ) ; 791 792 /* insertion sort */ 793 for ( i = 0; i < n; i++ ) { 794 AttributeDescription *ad = adv[i]; 795 for ( j = i-1; j>=0; j--) { 796 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break; 797 adv[j+1] = adv[j]; 798 } 799 adv[j+1] = ad; 800 } 801 } 802 803 for ( i = 0; adv[i]; i++ ) { 804 if ( mi->mi_attrs[i]->ai_desc != adv[i] ) { 805 for ( j = i+1; j < mi->mi_nattrs; j++ ) { 806 if ( mi->mi_attrs[j]->ai_desc == adv[i] ) { 807 AttrInfo *ai = mi->mi_attrs[i]; 808 mi->mi_attrs[i] = mi->mi_attrs[j]; 809 mi->mi_attrs[j] = ai; 810 break; 811 } 812 } 813 if ( j == mi->mi_nattrs ) { 814 Debug( LDAP_DEBUG_ANY, 815 LDAP_XSTRING(mdb_tool_entry_reindex) 816 ": no index configured for %s\n", 817 adv[i]->ad_cname.bv_val, 0, 0 ); 818 return -1; 819 } 820 } 821 } 822 mi->mi_nattrs = i; 823 } 824 825 e = mdb_tool_entry_get( be, id ); 826 827 if( e == NULL ) { 828 Debug( LDAP_DEBUG_ANY, 829 LDAP_XSTRING(mdb_tool_entry_reindex) 830 ": could not locate id=%ld\n", 831 (long) id, 0, 0 ); 832 return -1; 833 } 834 835 if ( !txi ) { 836 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi ); 837 if( rc != 0 ) { 838 Debug( LDAP_DEBUG_ANY, 839 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": " 840 "txn_begin failed: %s (%d)\n", 841 mdb_strerror(rc), rc, 0 ); 842 goto done; 843 } 844 } 845 846 if ( slapMode & SLAP_TRUNCATE_MODE ) { 847 int i; 848 for ( i=0; i < mi->mi_nattrs; i++ ) { 849 rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 ); 850 if ( rc ) { 851 Debug( LDAP_DEBUG_ANY, 852 LDAP_XSTRING(mdb_tool_entry_reindex) 853 ": (Truncate) mdb_drop(%s) failed: %s (%d)\n", 854 mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val, 855 mdb_strerror(rc), rc ); 856 return -1; 857 } 858 } 859 slapMode ^= SLAP_TRUNCATE_MODE; 860 } 861 862 /* 863 * just (re)add them for now 864 * Use truncate mode to empty/reset index databases 865 */ 866 867 Debug( LDAP_DEBUG_TRACE, 868 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n", 869 (long) id, 0, 0 ); 870 871 op.o_hdr = &ohdr; 872 op.o_bd = be; 873 op.o_tmpmemctx = NULL; 874 op.o_tmpmfuncs = &ch_mfuncs; 875 876 rc = mdb_tool_index_add( &op, txi, e ); 877 878 done: 879 if( rc == 0 ) { 880 mdb_writes++; 881 if ( mdb_writes >= mdb_writes_per_commit ) { 882 MDB_val key; 883 unsigned i; 884 MDB_TOOL_IDL_FLUSH( be, txi ); 885 rc = mdb_txn_commit( txi ); 886 mdb_writes = 0; 887 for ( i=0; i<mi->mi_nattrs; i++ ) 888 mi->mi_attrs[i]->ai_cursor = NULL; 889 if( rc != 0 ) { 890 Debug( LDAP_DEBUG_ANY, 891 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) 892 ": txn_commit failed: %s (%d)\n", 893 mdb_strerror(rc), rc, 0 ); 894 e->e_id = NOID; 895 } 896 mdb_cursor_close( cursor ); 897 txi = NULL; 898 /* Must close the read txn to allow old pages to be reclaimed. */ 899 mdb_txn_abort( txn ); 900 /* and then reopen it so that tool_entry_next still works. */ 901 mdb_txn_begin( mi->mi_dbenv, NULL, MDB_RDONLY, &txn ); 902 mdb_cursor_open( txn, mi->mi_id2entry, &cursor ); 903 key.mv_data = &id; 904 key.mv_size = sizeof(ID); 905 mdb_cursor_get( cursor, &key, NULL, MDB_SET ); 906 } 907 908 } else { 909 unsigned i; 910 mdb_writes = 0; 911 mdb_cursor_close( cursor ); 912 cursor = NULL; 913 mdb_txn_abort( txi ); 914 for ( i=0; i<mi->mi_nattrs; i++ ) 915 mi->mi_attrs[i]->ai_cursor = NULL; 916 Debug( LDAP_DEBUG_ANY, 917 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) 918 ": txn_aborted! err=%d\n", 919 rc, 0, 0 ); 920 e->e_id = NOID; 921 txi = NULL; 922 } 923 mdb_entry_release( &op, e, 0 ); 924 925 return rc; 926 } 927 928 ID mdb_tool_entry_modify( 929 BackendDB *be, 930 Entry *e, 931 struct berval *text ) 932 { 933 int rc; 934 struct mdb_info *mdb; 935 Operation op = {0}; 936 Opheader ohdr = {0}; 937 938 assert( be != NULL ); 939 assert( slapMode & SLAP_TOOL_MODE ); 940 941 assert( text != NULL ); 942 assert( text->bv_val != NULL ); 943 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 944 945 assert ( e->e_id != NOID ); 946 947 Debug( LDAP_DEBUG_TRACE, 948 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n", 949 (long) e->e_id, e->e_dn, 0 ); 950 951 mdb = (struct mdb_info *) be->be_private; 952 953 if( cursor ) { 954 mdb_cursor_close( cursor ); 955 cursor = NULL; 956 } 957 if ( !txn ) { 958 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn ); 959 if( rc != 0 ) { 960 snprintf( text->bv_val, text->bv_len, 961 "txn_begin failed: %s (%d)", 962 mdb_strerror(rc), rc ); 963 Debug( LDAP_DEBUG_ANY, 964 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n", 965 text->bv_val, 0, 0 ); 966 return NOID; 967 } 968 } 969 970 op.o_hdr = &ohdr; 971 op.o_bd = be; 972 op.o_tmpmemctx = NULL; 973 op.o_tmpmfuncs = &ch_mfuncs; 974 975 /* id2entry index */ 976 rc = mdb_id2entry_update( &op, txn, NULL, e ); 977 if( rc != 0 ) { 978 snprintf( text->bv_val, text->bv_len, 979 "id2entry_update failed: err=%d", rc ); 980 Debug( LDAP_DEBUG_ANY, 981 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n", 982 text->bv_val, 0, 0 ); 983 goto done; 984 } 985 986 done: 987 if( rc == 0 ) { 988 rc = mdb_txn_commit( txn ); 989 if( rc != 0 ) { 990 snprintf( text->bv_val, text->bv_len, 991 "txn_commit failed: %s (%d)", 992 mdb_strerror(rc), rc ); 993 Debug( LDAP_DEBUG_ANY, 994 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": " 995 "%s\n", text->bv_val, 0, 0 ); 996 e->e_id = NOID; 997 } 998 999 } else { 1000 mdb_txn_abort( txn ); 1001 snprintf( text->bv_val, text->bv_len, 1002 "txn_aborted! %s (%d)", 1003 mdb_strerror(rc), rc ); 1004 Debug( LDAP_DEBUG_ANY, 1005 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n", 1006 text->bv_val, 0, 0 ); 1007 e->e_id = NOID; 1008 } 1009 txn = NULL; 1010 idcursor = NULL; 1011 1012 return e->e_id; 1013 } 1014 1015 static void * 1016 mdb_tool_index_task( void *ctx, void *ptr ) 1017 { 1018 int base = *(int *)ptr; 1019 1020 free( ptr ); 1021 while ( 1 ) { 1022 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 1023 mdb_tool_index_tcount--; 1024 if ( !mdb_tool_index_tcount ) 1025 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main ); 1026 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work, 1027 &mdb_tool_index_mutex ); 1028 if ( slapd_shutdown ) { 1029 mdb_tool_index_tcount--; 1030 if ( !mdb_tool_index_tcount ) 1031 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main ); 1032 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 1033 break; 1034 } 1035 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 1036 mdb_tool_index_rec[base].ir_i = mdb_index_recrun( mdb_tool_ix_op, 1037 mdb_tool_ix_txn, 1038 mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base ); 1039 } 1040 1041 return NULL; 1042 } 1043 1044 #ifdef MDB_TOOL_IDL_CACHING 1045 static int 1046 mdb_tool_idl_cmp( const void *v1, const void *v2 ) 1047 { 1048 const mdb_tool_idl_cache *c1 = v1, *c2 = v2; 1049 int rc; 1050 1051 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc; 1052 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len ); 1053 } 1054 1055 static int 1056 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic ) 1057 { 1058 mdb_tool_idl_cache_entry *ice; 1059 MDB_val key, data[2]; 1060 int i, rc; 1061 ID id, nid; 1062 1063 /* Freshly allocated, ignore it */ 1064 if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) { 1065 return 0; 1066 } 1067 1068 key.mv_data = ic->kstr.bv_val; 1069 key.mv_size = ic->kstr.bv_len; 1070 1071 if ( ic->count > MDB_IDL_DB_SIZE ) { 1072 while ( ic->flags & WAS_FOUND ) { 1073 rc = mdb_cursor_get( mc, &key, data, MDB_SET ); 1074 if ( rc ) { 1075 /* FIXME: find out why this happens */ 1076 ic->flags = 0; 1077 break; 1078 } 1079 if ( ic->flags & WAS_RANGE ) { 1080 /* Skip lo */ 1081 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); 1082 1083 /* Get hi */ 1084 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); 1085 1086 /* Store range hi */ 1087 data[0].mv_data = &ic->last; 1088 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT ); 1089 } else { 1090 /* Delete old data, replace with range */ 1091 ic->first = *(ID *)data[0].mv_data; 1092 mdb_cursor_del( mc, MDB_NODUPDATA ); 1093 } 1094 break; 1095 } 1096 if ( !(ic->flags & WAS_RANGE)) { 1097 /* range, didn't exist before */ 1098 nid = 0; 1099 data[0].mv_size = sizeof(ID); 1100 data[0].mv_data = &nid; 1101 rc = mdb_cursor_put( mc, &key, data, 0 ); 1102 if ( rc == 0 ) { 1103 data[0].mv_data = &ic->first; 1104 rc = mdb_cursor_put( mc, &key, data, 0 ); 1105 if ( rc == 0 ) { 1106 data[0].mv_data = &ic->last; 1107 rc = mdb_cursor_put( mc, &key, data, 0 ); 1108 } 1109 } 1110 if ( rc ) { 1111 rc = -1; 1112 } 1113 } 1114 } else { 1115 /* Normal write */ 1116 int n; 1117 1118 data[0].mv_size = sizeof(ID); 1119 rc = 0; 1120 i = ic->offset; 1121 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) { 1122 int end; 1123 if ( ice->next ) { 1124 end = IDBLOCK; 1125 } else { 1126 end = ic->count & (IDBLOCK-1); 1127 if ( !end ) 1128 end = IDBLOCK; 1129 } 1130 data[1].mv_size = end - i; 1131 data[0].mv_data = &ice->ids[i]; 1132 i = 0; 1133 rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE ); 1134 if ( rc ) { 1135 if ( rc == MDB_KEYEXIST ) { 1136 rc = 0; 1137 continue; 1138 } 1139 rc = -1; 1140 break; 1141 } 1142 } 1143 if ( ic->head ) { 1144 ic->tail->next = ai->ai_flist; 1145 ai->ai_flist = ic->head; 1146 } 1147 } 1148 ic->head = ai->ai_clist; 1149 ai->ai_clist = ic; 1150 return rc; 1151 } 1152 1153 static int 1154 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai ) 1155 { 1156 MDB_cursor *mc; 1157 Avlnode *root; 1158 int rc; 1159 1160 mdb_cursor_open( txn, ai->ai_dbi, &mc ); 1161 root = tavl_end( ai->ai_root, TAVL_DIR_LEFT ); 1162 do { 1163 rc = mdb_tool_idl_flush_one( mc, ai, root->avl_data ); 1164 if ( rc != -1 ) 1165 rc = 0; 1166 } while ((root = tavl_next(root, TAVL_DIR_RIGHT))); 1167 mdb_cursor_close( mc ); 1168 1169 return rc; 1170 } 1171 1172 static int 1173 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn ) 1174 { 1175 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 1176 int rc = 0; 1177 unsigned int i, dbi; 1178 1179 for ( i=0; i < mdb->mi_nattrs; i++ ) { 1180 if ( !mdb->mi_attrs[i]->ai_root ) continue; 1181 rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i] ); 1182 tavl_free(mdb->mi_attrs[i]->ai_root, NULL); 1183 mdb->mi_attrs[i]->ai_root = NULL; 1184 if ( rc ) 1185 break; 1186 } 1187 return rc; 1188 } 1189 1190 int mdb_tool_idl_add( 1191 BackendDB *be, 1192 MDB_cursor *mc, 1193 struct berval *keys, 1194 ID id ) 1195 { 1196 MDB_dbi dbi; 1197 mdb_tool_idl_cache *ic, itmp; 1198 mdb_tool_idl_cache_entry *ice; 1199 int i, rc, lcount; 1200 AttrInfo *ai = (AttrInfo *)mc; 1201 mc = ai->ai_cursor; 1202 1203 dbi = ai->ai_dbi; 1204 for (i=0; keys[i].bv_val; i++) { 1205 itmp.kstr = keys[i]; 1206 ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp ); 1207 1208 /* No entry yet, create one */ 1209 if ( !ic ) { 1210 MDB_val key, data; 1211 ID nid; 1212 int rc; 1213 1214 if ( ai->ai_clist ) { 1215 ic = ai->ai_clist; 1216 ai->ai_clist = ic->head; 1217 } else { 1218 ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 ); 1219 } 1220 ic->kstr.bv_len = itmp.kstr.bv_len; 1221 ic->kstr.bv_val = (char *)(ic+1); 1222 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len ); 1223 ic->head = ic->tail = NULL; 1224 ic->last = 0; 1225 ic->count = 0; 1226 ic->offset = 0; 1227 ic->flags = 0; 1228 tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp, 1229 avl_dup_error ); 1230 1231 /* load existing key count here */ 1232 key.mv_size = keys[i].bv_len; 1233 key.mv_data = keys[i].bv_val; 1234 rc = mdb_cursor_get( mc, &key, &data, MDB_SET ); 1235 if ( rc == 0 ) { 1236 ic->flags |= WAS_FOUND; 1237 nid = *(ID *)data.mv_data; 1238 if ( nid == 0 ) { 1239 ic->count = MDB_IDL_DB_SIZE+1; 1240 ic->flags |= WAS_RANGE; 1241 } else { 1242 size_t count; 1243 1244 mdb_cursor_count( mc, &count ); 1245 ic->count = count; 1246 ic->first = nid; 1247 ic->offset = count & (IDBLOCK-1); 1248 } 1249 } 1250 } 1251 /* are we a range already? */ 1252 if ( ic->count > MDB_IDL_DB_SIZE ) { 1253 ic->last = id; 1254 continue; 1255 /* Are we at the limit, and converting to a range? */ 1256 } else if ( ic->count == MDB_IDL_DB_SIZE ) { 1257 if ( ic->head ) { 1258 ic->tail->next = ai->ai_flist; 1259 ai->ai_flist = ic->head; 1260 } 1261 ic->head = ic->tail = NULL; 1262 ic->last = id; 1263 ic->count++; 1264 continue; 1265 } 1266 /* No free block, create that too */ 1267 lcount = ic->count & (IDBLOCK-1); 1268 if ( !ic->tail || lcount == 0) { 1269 if ( ai->ai_flist ) { 1270 ice = ai->ai_flist; 1271 ai->ai_flist = ice->next; 1272 } else { 1273 ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry )); 1274 } 1275 ice->next = NULL; 1276 if ( !ic->head ) { 1277 ic->head = ice; 1278 } else { 1279 ic->tail->next = ice; 1280 } 1281 ic->tail = ice; 1282 if ( lcount ) 1283 ice->ids[lcount-1] = 0; 1284 if ( !ic->count ) 1285 ic->first = id; 1286 } 1287 ice = ic->tail; 1288 if (!lcount || ice->ids[lcount-1] != id) 1289 ice->ids[lcount] = id; 1290 ic->count++; 1291 } 1292 1293 return 0; 1294 } 1295 #endif /* MDB_TOOL_IDL_CACHING */ 1296 1297 /* Upgrade from pre 2.4.34 dn2id format */ 1298 1299 #include <ac/unistd.h> 1300 #include <lutil_meter.h> 1301 1302 #define STACKSIZ 2048 1303 1304 typedef struct rec { 1305 ID id; 1306 size_t len; 1307 char rdn[512]; 1308 } rec; 1309 1310 static int 1311 mdb_dn2id_upgrade( BackendDB *be ) { 1312 struct mdb_info *mi = (struct mdb_info *) be->be_private; 1313 MDB_txn *mt; 1314 MDB_cursor *mc = NULL; 1315 MDB_val key, data; 1316 char *ptr; 1317 int rc, writes=0, depth=0; 1318 int enable_meter = 0; 1319 ID id = 0, *num, count = 0; 1320 rec *stack; 1321 lutil_meter_t meter; 1322 1323 if (!(mi->mi_flags & MDB_NEED_UPGRADE)) { 1324 Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n", 1325 be->be_suffix[0].bv_val, 0, 0 ); 1326 return 0; 1327 } 1328 1329 { 1330 MDB_stat st; 1331 1332 mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st); 1333 if (!st.ms_entries) { 1334 /* Empty DB, nothing to upgrade? */ 1335 return 0; 1336 } 1337 if (isatty(2)) 1338 enable_meter = !lutil_meter_open(&meter, 1339 &lutil_meter_text_display, 1340 &lutil_meter_linear_estimator, 1341 st.ms_entries); 1342 } 1343 1344 num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec))); 1345 stack = (rec *)(num + STACKSIZ); 1346 1347 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); 1348 if (rc) { 1349 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n", 1350 mdb_strerror(rc), rc, 0 ); 1351 goto leave; 1352 } 1353 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); 1354 if (rc) { 1355 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n", 1356 mdb_strerror(rc), rc, 0 ); 1357 goto leave; 1358 } 1359 1360 key.mv_size = sizeof(ID); 1361 /* post-order depth-first update */ 1362 for(;;) { 1363 size_t dkids; 1364 unsigned char *ptr; 1365 1366 /* visit */ 1367 key.mv_data = &id; 1368 stack[depth].id = id; 1369 rc = mdb_cursor_get(mc, &key, &data, MDB_SET); 1370 if (rc) { 1371 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n", 1372 mdb_strerror(rc), rc, 0 ); 1373 goto leave; 1374 } 1375 num[depth] = 1; 1376 1377 rc = mdb_cursor_count(mc, &dkids); 1378 if (rc) { 1379 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n", 1380 mdb_strerror(rc), rc, 0 ); 1381 goto leave; 1382 } 1383 if (dkids > 1) { 1384 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); 1385 down: 1386 ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID); 1387 memcpy(&id, ptr, sizeof(ID)); 1388 depth++; 1389 memcpy(stack[depth].rdn, data.mv_data, data.mv_size); 1390 stack[depth].len = data.mv_size; 1391 continue; 1392 } 1393 1394 1395 /* pop: write updated count, advance to next node */ 1396 pop: 1397 /* update superior counts */ 1398 if (depth) 1399 num[depth-1] += num[depth]; 1400 1401 key.mv_data = &id; 1402 id = stack[depth-1].id; 1403 data.mv_data = stack[depth].rdn; 1404 data.mv_size = stack[depth].len; 1405 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); 1406 if (rc) { 1407 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n", 1408 mdb_strerror(rc), rc, 0 ); 1409 goto leave; 1410 } 1411 data.mv_data = stack[depth].rdn; 1412 ptr = (unsigned char *)data.mv_data + data.mv_size; 1413 memcpy(ptr, &num[depth], sizeof(ID)); 1414 data.mv_size += sizeof(ID); 1415 rc = mdb_cursor_del(mc, 0); 1416 if (rc) { 1417 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n", 1418 mdb_strerror(rc), rc, 0 ); 1419 goto leave; 1420 } 1421 rc = mdb_cursor_put(mc, &key, &data, 0); 1422 if (rc) { 1423 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n", 1424 mdb_strerror(rc), rc, 0 ); 1425 goto leave; 1426 } 1427 count++; 1428 #if 1 1429 if (enable_meter) 1430 lutil_meter_update(&meter, count, 0); 1431 #else 1432 { 1433 int len; 1434 ptr = data.mv_data; 1435 len = (ptr[0] & 0x7f) << 8 | ptr[1]; 1436 printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2); 1437 } 1438 #endif 1439 writes++; 1440 if (writes == 1000) { 1441 mdb_cursor_close(mc); 1442 rc = mdb_txn_commit(mt); 1443 if (rc) { 1444 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n", 1445 mdb_strerror(rc), rc, 0 ); 1446 goto leave; 1447 } 1448 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); 1449 if (rc) { 1450 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n", 1451 mdb_strerror(rc), rc, 0 ); 1452 goto leave; 1453 } 1454 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); 1455 if (rc) { 1456 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n", 1457 mdb_strerror(rc), rc, 0 ); 1458 goto leave; 1459 } 1460 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); 1461 if (rc) { 1462 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n", 1463 mdb_strerror(rc), rc, 0 ); 1464 goto leave; 1465 } 1466 writes = 0; 1467 } 1468 depth--; 1469 1470 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); 1471 if (rc == 0) 1472 goto down; 1473 rc = 0; 1474 if (depth) 1475 goto pop; 1476 else 1477 break; 1478 } 1479 leave: 1480 mdb_cursor_close(mc); 1481 if (mt) { 1482 int r2; 1483 r2 = mdb_txn_commit(mt); 1484 if (r2) { 1485 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n", 1486 mdb_strerror(r2), r2, 0 ); 1487 if (!rc) 1488 rc = r2; 1489 } 1490 } 1491 ch_free(num); 1492 if (enable_meter) { 1493 lutil_meter_update(&meter, count, 1); 1494 lutil_meter_close(&meter); 1495 } 1496 return rc; 1497 } 1498