1 /* $NetBSD: tools.c,v 1.1.1.3 2018/02/06 01:53:17 christos Exp $ */ 2 3 /* tools.c - tools for slap tools */ 4 /* $OpenLDAP$ */ 5 /* This work is part of OpenLDAP Software <http://www.openldap.org/>. 6 * 7 * Copyright 2011-2017 The OpenLDAP Foundation. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted only as authorized by the OpenLDAP 12 * Public License. 13 * 14 * A copy of this license is available in the file LICENSE in the 15 * top-level directory of the distribution or, alternatively, at 16 * <http://www.OpenLDAP.org/license.html>. 17 */ 18 19 #include <sys/cdefs.h> 20 __RCSID("$NetBSD: tools.c,v 1.1.1.3 2018/02/06 01:53:17 christos Exp $"); 21 22 #include "portable.h" 23 24 #include <stdio.h> 25 #include <ac/string.h> 26 #include <ac/errno.h> 27 28 #define AVL_INTERNAL 29 #include "back-mdb.h" 30 #include "idl.h" 31 32 #ifdef MDB_TOOL_IDL_CACHING 33 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn ); 34 35 #define IDBLOCK 1024 36 37 typedef struct mdb_tool_idl_cache_entry { 38 struct mdb_tool_idl_cache_entry *next; 39 ID ids[IDBLOCK]; 40 } mdb_tool_idl_cache_entry; 41 42 typedef struct mdb_tool_idl_cache { 43 struct berval kstr; 44 mdb_tool_idl_cache_entry *head, *tail; 45 ID first, last; 46 int count; 47 short offset; 48 short flags; 49 } mdb_tool_idl_cache; 50 #define WAS_FOUND 0x01 51 #define WAS_RANGE 0x02 52 53 #define MDB_TOOL_IDL_FLUSH(be, txn) mdb_tool_idl_flush(be, txn) 54 #else 55 #define MDB_TOOL_IDL_FLUSH(be, txn) 56 #endif /* MDB_TOOL_IDL_CACHING */ 57 58 MDB_txn *mdb_tool_txn = NULL; 59 60 static MDB_txn *txi = NULL; 61 static MDB_cursor *cursor = NULL, *idcursor = NULL; 62 static MDB_cursor *mcp = NULL, *mcd = NULL; 63 static MDB_val key, data; 64 static ID previd = NOID; 65 66 typedef struct dn_id { 67 ID id; 68 struct berval dn; 69 } dn_id; 70 71 #define HOLE_SIZE 4096 72 static dn_id hbuf[HOLE_SIZE], *holes = hbuf; 73 static unsigned nhmax = HOLE_SIZE; 74 static unsigned nholes; 75 76 static struct berval *tool_base; 77 static int tool_scope; 78 static Filter *tool_filter; 79 static Entry *tool_next_entry; 80 81 static ID mdb_tool_ix_id; 82 static Operation *mdb_tool_ix_op; 83 static MDB_txn *mdb_tool_ix_txn; 84 static int mdb_tool_index_tcount, mdb_tool_threads; 85 static IndexRec *mdb_tool_index_rec; 86 static struct mdb_info *mdb_tool_info; 87 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex; 88 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main; 89 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work; 90 static void * mdb_tool_index_task( void *ctx, void *ptr ); 91 92 static int mdb_writes, mdb_writes_per_commit; 93 94 /* Number of ops per commit in Quick mode. 95 * Batching speeds writes overall, but too large a 96 * batch will fail with MDB_TXN_FULL. 97 */ 98 #ifndef MDB_WRITES_PER_COMMIT 99 #define MDB_WRITES_PER_COMMIT 500 100 #endif 101 102 static int 103 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ); 104 105 int mdb_tool_entry_open( 106 BackendDB *be, int mode ) 107 { 108 /* In Quick mode, commit once per 500 entries */ 109 mdb_writes = 0; 110 if ( slapMode & SLAP_TOOL_QUICK ) 111 mdb_writes_per_commit = MDB_WRITES_PER_COMMIT; 112 else 113 mdb_writes_per_commit = 1; 114 115 /* Set up for threaded slapindex */ 116 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) { 117 if ( !mdb_tool_info ) { 118 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 119 ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex ); 120 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main ); 121 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work ); 122 if ( mdb->mi_nattrs ) { 123 int i; 124 #if 0 /* threaded indexing has no performance advantage */ 125 mdb_tool_threads = slap_tool_thread_max - 1; 126 #endif 127 if ( mdb_tool_threads > 1 ) { 128 mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec )); 129 mdb_tool_index_tcount = mdb_tool_threads - 1; 130 for (i=1; i<mdb_tool_threads; i++) { 131 int *ptr = ch_malloc( sizeof( int )); 132 *ptr = i; 133 ldap_pvt_thread_pool_submit( &connection_pool, 134 mdb_tool_index_task, ptr ); 135 } 136 mdb_tool_info = mdb; 137 } 138 } 139 } 140 } 141 142 return 0; 143 } 144 145 int mdb_tool_entry_close( 146 BackendDB *be ) 147 { 148 if ( mdb_tool_info ) { 149 slapd_shutdown = 1; 150 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 151 152 /* There might still be some threads starting */ 153 while ( mdb_tool_index_tcount > 0 ) { 154 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 155 &mdb_tool_index_mutex ); 156 } 157 158 mdb_tool_index_tcount = mdb_tool_threads - 1; 159 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work ); 160 161 /* Make sure all threads are stopped */ 162 while ( mdb_tool_index_tcount > 0 ) { 163 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 164 &mdb_tool_index_mutex ); 165 } 166 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 167 168 mdb_tool_info = NULL; 169 slapd_shutdown = 0; 170 ch_free( mdb_tool_index_rec ); 171 mdb_tool_index_tcount = mdb_tool_threads - 1; 172 } 173 174 if( idcursor ) { 175 mdb_cursor_close( idcursor ); 176 idcursor = NULL; 177 } 178 if( cursor ) { 179 mdb_cursor_close( cursor ); 180 cursor = NULL; 181 } 182 { 183 struct mdb_info *mdb = be->be_private; 184 if ( mdb ) { 185 int i; 186 for (i=0; i<mdb->mi_nattrs; i++) 187 mdb->mi_attrs[i]->ai_cursor = NULL; 188 } 189 } 190 if( mdb_tool_txn ) { 191 int rc; 192 MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn ); 193 if (( rc = mdb_txn_commit( mdb_tool_txn ))) { 194 Debug( LDAP_DEBUG_ANY, 195 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: " 196 "txn_commit failed: %s (%d)\n", 197 be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); 198 return -1; 199 } 200 mdb_tool_txn = NULL; 201 } 202 203 if( nholes ) { 204 unsigned i; 205 fprintf( stderr, "Error, entries missing!\n"); 206 for (i=0; i<nholes; i++) { 207 fprintf(stderr, " entry %ld: %s\n", 208 holes[i].id, holes[i].dn.bv_val); 209 } 210 nholes = 0; 211 return -1; 212 } 213 214 return 0; 215 } 216 217 ID 218 mdb_tool_entry_first_x( 219 BackendDB *be, 220 struct berval *base, 221 int scope, 222 Filter *f ) 223 { 224 tool_base = base; 225 tool_scope = scope; 226 tool_filter = f; 227 228 return mdb_tool_entry_next( be ); 229 } 230 231 ID mdb_tool_entry_next( 232 BackendDB *be ) 233 { 234 int rc; 235 ID id; 236 struct mdb_info *mdb; 237 238 assert( be != NULL ); 239 assert( slapMode & SLAP_TOOL_MODE ); 240 241 mdb = (struct mdb_info *) be->be_private; 242 assert( mdb != NULL ); 243 244 if ( !mdb_tool_txn ) { 245 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn ); 246 if ( rc ) 247 return NOID; 248 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor ); 249 if ( rc ) { 250 mdb_txn_abort( mdb_tool_txn ); 251 return NOID; 252 } 253 } 254 255 next:; 256 rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT ); 257 258 if( rc ) { 259 return NOID; 260 } 261 262 previd = *(ID *)key.mv_data; 263 id = previd; 264 265 if ( !data.mv_size ) 266 goto next; 267 268 if ( tool_filter || tool_base ) { 269 static Operation op = {0}; 270 static Opheader ohdr = {0}; 271 272 op.o_hdr = &ohdr; 273 op.o_bd = be; 274 op.o_tmpmemctx = NULL; 275 op.o_tmpmfuncs = &ch_mfuncs; 276 277 if ( tool_next_entry ) { 278 mdb_entry_release( &op, tool_next_entry, 0 ); 279 tool_next_entry = NULL; 280 } 281 282 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry ); 283 if ( rc == LDAP_NO_SUCH_OBJECT ) { 284 goto next; 285 } 286 287 assert( tool_next_entry != NULL ); 288 289 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE ) 290 { 291 mdb_entry_release( &op, tool_next_entry, 0 ); 292 tool_next_entry = NULL; 293 goto next; 294 } 295 } 296 297 return id; 298 } 299 300 ID mdb_tool_dn2id_get( 301 Backend *be, 302 struct berval *dn 303 ) 304 { 305 struct mdb_info *mdb; 306 Operation op = {0}; 307 Opheader ohdr = {0}; 308 ID id; 309 int rc; 310 311 if ( BER_BVISEMPTY(dn) ) 312 return 0; 313 314 mdb = (struct mdb_info *) be->be_private; 315 316 if ( !mdb_tool_txn ) { 317 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ? 318 MDB_RDONLY : 0, &mdb_tool_txn ); 319 if ( rc ) 320 return NOID; 321 } 322 323 op.o_hdr = &ohdr; 324 op.o_bd = be; 325 op.o_tmpmemctx = NULL; 326 op.o_tmpmfuncs = &ch_mfuncs; 327 328 rc = mdb_dn2id( &op, mdb_tool_txn, NULL, dn, &id, NULL, NULL, NULL ); 329 if ( rc == MDB_NOTFOUND ) 330 return NOID; 331 332 return id; 333 } 334 335 static int 336 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ) 337 { 338 Operation op = {0}; 339 Opheader ohdr = {0}; 340 341 Entry *e = NULL; 342 struct berval dn = BER_BVNULL, ndn = BER_BVNULL; 343 int rc; 344 345 assert( be != NULL ); 346 assert( slapMode & SLAP_TOOL_MODE ); 347 348 if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) { 349 *ep = tool_next_entry; 350 tool_next_entry = NULL; 351 return LDAP_SUCCESS; 352 } 353 354 if ( id != previd ) { 355 key.mv_size = sizeof(ID); 356 key.mv_data = &id; 357 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET ); 358 if ( rc ) { 359 rc = LDAP_OTHER; 360 goto done; 361 } 362 } 363 if ( !data.mv_size ) { 364 rc = LDAP_NO_SUCH_OBJECT; 365 goto done; 366 } 367 368 op.o_hdr = &ohdr; 369 op.o_bd = be; 370 op.o_tmpmemctx = NULL; 371 op.o_tmpmfuncs = &ch_mfuncs; 372 if ( slapMode & SLAP_TOOL_READONLY ) { 373 rc = mdb_id2name( &op, mdb_tool_txn, &idcursor, id, &dn, &ndn ); 374 if ( rc ) { 375 rc = LDAP_OTHER; 376 goto done; 377 } 378 if ( tool_base != NULL ) { 379 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) { 380 ch_free( dn.bv_val ); 381 ch_free( ndn.bv_val ); 382 rc = LDAP_NO_SUCH_OBJECT; 383 goto done; 384 } 385 } 386 } 387 rc = mdb_entry_decode( &op, mdb_tool_txn, &data, &e ); 388 e->e_id = id; 389 if ( !BER_BVISNULL( &dn )) { 390 e->e_name = dn; 391 e->e_nname = ndn; 392 } else { 393 e->e_name.bv_val = NULL; 394 e->e_nname.bv_val = NULL; 395 } 396 397 done: 398 if ( e != NULL ) { 399 *ep = e; 400 } 401 402 return rc; 403 } 404 405 Entry* 406 mdb_tool_entry_get( BackendDB *be, ID id ) 407 { 408 Entry *e = NULL; 409 int rc; 410 411 if ( !mdb_tool_txn ) { 412 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 413 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 414 (slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &mdb_tool_txn ); 415 if ( rc ) 416 return NULL; 417 } 418 if ( !cursor ) { 419 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 420 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor ); 421 if ( rc ) { 422 mdb_txn_abort( mdb_tool_txn ); 423 mdb_tool_txn = NULL; 424 return NULL; 425 } 426 } 427 (void)mdb_tool_entry_get_int( be, id, &e ); 428 return e; 429 } 430 431 static int mdb_tool_next_id( 432 Operation *op, 433 MDB_txn *tid, 434 Entry *e, 435 struct berval *text, 436 int hole ) 437 { 438 struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; 439 struct berval dn = e->e_name; 440 struct berval ndn = e->e_nname; 441 struct berval pdn, npdn, nmatched; 442 ID id, pid = 0; 443 int rc; 444 445 if (ndn.bv_len == 0) { 446 e->e_id = 0; 447 return 0; 448 } 449 450 rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, NULL, &nmatched ); 451 if ( rc == MDB_NOTFOUND ) { 452 if ( !be_issuffix( op->o_bd, &ndn ) ) { 453 ID eid = e->e_id; 454 dnParent( &ndn, &npdn ); 455 if ( nmatched.bv_len != npdn.bv_len ) { 456 dnParent( &dn, &pdn ); 457 e->e_name = pdn; 458 e->e_nname = npdn; 459 rc = mdb_tool_next_id( op, tid, e, text, 1 ); 460 e->e_name = dn; 461 e->e_nname = ndn; 462 if ( rc ) { 463 return rc; 464 } 465 /* If parent didn't exist, it was created just now 466 * and its ID is now in e->e_id. Make sure the current 467 * entry gets added under the new parent ID. 468 */ 469 if ( eid != e->e_id ) { 470 pid = e->e_id; 471 } 472 } else { 473 pid = id; 474 } 475 } 476 rc = mdb_next_id( op->o_bd, idcursor, &e->e_id ); 477 if ( rc ) { 478 snprintf( text->bv_val, text->bv_len, 479 "next_id failed: %s (%d)", 480 mdb_strerror(rc), rc ); 481 Debug( LDAP_DEBUG_ANY, 482 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 483 return rc; 484 } 485 rc = mdb_dn2id_add( op, mcp, mcd, pid, 1, 1, e ); 486 if ( rc ) { 487 snprintf( text->bv_val, text->bv_len, 488 "dn2id_add failed: %s (%d)", 489 mdb_strerror(rc), rc ); 490 Debug( LDAP_DEBUG_ANY, 491 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 492 } else if ( hole ) { 493 MDB_val key, data; 494 if ( nholes == nhmax - 1 ) { 495 if ( holes == hbuf ) { 496 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 ); 497 AC_MEMCPY( holes, hbuf, sizeof(hbuf) ); 498 } else { 499 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 ); 500 } 501 nhmax *= 2; 502 } 503 ber_dupbv( &holes[nholes].dn, &ndn ); 504 holes[nholes++].id = e->e_id; 505 key.mv_size = sizeof(ID); 506 key.mv_data = &e->e_id; 507 data.mv_size = 0; 508 data.mv_data = NULL; 509 rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE ); 510 if ( rc == MDB_KEYEXIST ) 511 rc = 0; 512 if ( rc ) { 513 snprintf( text->bv_val, text->bv_len, 514 "dummy id2entry add failed: %s (%d)", 515 mdb_strerror(rc), rc ); 516 Debug( LDAP_DEBUG_ANY, 517 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 518 } 519 } 520 } else if ( !hole ) { 521 unsigned i, j; 522 523 e->e_id = id; 524 525 for ( i=0; i<nholes; i++) { 526 if ( holes[i].id == e->e_id ) { 527 free(holes[i].dn.bv_val); 528 for (j=i;j<nholes;j++) holes[j] = holes[j+1]; 529 holes[j].id = 0; 530 nholes--; 531 break; 532 } else if ( holes[i].id > e->e_id ) { 533 break; 534 } 535 } 536 } 537 return rc; 538 } 539 540 static int 541 mdb_tool_index_add( 542 Operation *op, 543 MDB_txn *txn, 544 Entry *e ) 545 { 546 struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; 547 548 if ( !mdb->mi_nattrs ) 549 return 0; 550 551 if ( mdb_tool_threads > 1 ) { 552 IndexRec *ir; 553 int i, rc; 554 Attribute *a; 555 556 ir = mdb_tool_index_rec; 557 for (i=0; i<mdb->mi_nattrs; i++) 558 ir[i].ir_attrs = NULL; 559 560 for ( a = e->e_attrs; a != NULL; a = a->a_next ) { 561 rc = mdb_index_recset( mdb, a, a->a_desc->ad_type, 562 &a->a_desc->ad_tags, ir ); 563 if ( rc ) 564 return rc; 565 } 566 for (i=0; i<mdb->mi_nattrs; i++) { 567 if ( !ir[i].ir_ai ) 568 break; 569 rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi, 570 &ir[i].ir_ai->ai_cursor ); 571 if ( rc ) 572 return rc; 573 } 574 mdb_tool_ix_id = e->e_id; 575 mdb_tool_ix_op = op; 576 mdb_tool_ix_txn = txn; 577 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 578 /* Wait for all threads to be ready */ 579 while ( mdb_tool_index_tcount ) { 580 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 581 &mdb_tool_index_mutex ); 582 } 583 584 for ( i=1; i<mdb_tool_threads; i++ ) 585 mdb_tool_index_rec[i].ir_i = LDAP_BUSY; 586 mdb_tool_index_tcount = mdb_tool_threads - 1; 587 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 588 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work ); 589 590 rc = mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 ); 591 if ( rc ) 592 return rc; 593 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 594 for ( i=1; i<mdb_tool_threads; i++ ) { 595 if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) { 596 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main, 597 &mdb_tool_index_mutex ); 598 i--; 599 continue; 600 } 601 if ( mdb_tool_index_rec[i].ir_i ) { 602 rc = mdb_tool_index_rec[i].ir_i; 603 break; 604 } 605 } 606 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 607 return rc; 608 } else 609 { 610 return mdb_index_entry_add( op, txn, e ); 611 } 612 } 613 614 ID mdb_tool_entry_put( 615 BackendDB *be, 616 Entry *e, 617 struct berval *text ) 618 { 619 int rc; 620 struct mdb_info *mdb; 621 Operation op = {0}; 622 Opheader ohdr = {0}; 623 624 assert( be != NULL ); 625 assert( slapMode & SLAP_TOOL_MODE ); 626 627 assert( text != NULL ); 628 assert( text->bv_val != NULL ); 629 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 630 631 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put) 632 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 ); 633 634 mdb = (struct mdb_info *) be->be_private; 635 636 if ( !mdb_tool_txn ) { 637 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn ); 638 if( rc != 0 ) { 639 snprintf( text->bv_val, text->bv_len, 640 "txn_begin failed: %s (%d)", 641 mdb_strerror(rc), rc ); 642 Debug( LDAP_DEBUG_ANY, 643 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 644 text->bv_val, 0, 0 ); 645 return NOID; 646 } 647 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &idcursor ); 648 if( rc != 0 ) { 649 snprintf( text->bv_val, text->bv_len, 650 "cursor_open failed: %s (%d)", 651 mdb_strerror(rc), rc ); 652 Debug( LDAP_DEBUG_ANY, 653 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 654 text->bv_val, 0, 0 ); 655 return NOID; 656 } 657 if ( !mdb->mi_nextid ) { 658 ID dummy; 659 mdb_next_id( be, idcursor, &dummy ); 660 } 661 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcp ); 662 if( rc != 0 ) { 663 snprintf( text->bv_val, text->bv_len, 664 "cursor_open failed: %s (%d)", 665 mdb_strerror(rc), rc ); 666 Debug( LDAP_DEBUG_ANY, 667 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 668 text->bv_val, 0, 0 ); 669 return NOID; 670 } 671 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcd ); 672 if( rc != 0 ) { 673 snprintf( text->bv_val, text->bv_len, 674 "cursor_open failed: %s (%d)", 675 mdb_strerror(rc), rc ); 676 Debug( LDAP_DEBUG_ANY, 677 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 678 text->bv_val, 0, 0 ); 679 return NOID; 680 } 681 } 682 683 op.o_hdr = &ohdr; 684 op.o_bd = be; 685 op.o_tmpmemctx = NULL; 686 op.o_tmpmfuncs = &ch_mfuncs; 687 688 /* add dn2id indices */ 689 rc = mdb_tool_next_id( &op, mdb_tool_txn, e, text, 0 ); 690 if( rc != 0 ) { 691 goto done; 692 } 693 694 rc = mdb_tool_index_add( &op, mdb_tool_txn, e ); 695 if( rc != 0 ) { 696 snprintf( text->bv_val, text->bv_len, 697 "index_entry_add failed: err=%d", rc ); 698 Debug( LDAP_DEBUG_ANY, 699 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 700 text->bv_val, 0, 0 ); 701 goto done; 702 } 703 704 705 /* id2entry index */ 706 rc = mdb_id2entry_add( &op, mdb_tool_txn, idcursor, e ); 707 if( rc != 0 ) { 708 snprintf( text->bv_val, text->bv_len, 709 "id2entry_add failed: err=%d", rc ); 710 Debug( LDAP_DEBUG_ANY, 711 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 712 text->bv_val, 0, 0 ); 713 goto done; 714 } 715 716 done: 717 if( rc == 0 ) { 718 mdb_writes++; 719 if ( mdb_writes >= mdb_writes_per_commit ) { 720 unsigned i; 721 MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn ); 722 rc = mdb_txn_commit( mdb_tool_txn ); 723 for ( i=0; i<mdb->mi_nattrs; i++ ) 724 mdb->mi_attrs[i]->ai_cursor = NULL; 725 mdb_writes = 0; 726 mdb_tool_txn = NULL; 727 idcursor = NULL; 728 if( rc != 0 ) { 729 mdb->mi_numads = 0; 730 snprintf( text->bv_val, text->bv_len, 731 "txn_commit failed: %s (%d)", 732 mdb_strerror(rc), rc ); 733 Debug( LDAP_DEBUG_ANY, 734 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 735 text->bv_val, 0, 0 ); 736 e->e_id = NOID; 737 } 738 } 739 740 } else { 741 unsigned i; 742 mdb_txn_abort( mdb_tool_txn ); 743 mdb_tool_txn = NULL; 744 idcursor = NULL; 745 for ( i=0; i<mdb->mi_nattrs; i++ ) 746 mdb->mi_attrs[i]->ai_cursor = NULL; 747 mdb_writes = 0; 748 snprintf( text->bv_val, text->bv_len, 749 "txn_aborted! %s (%d)", 750 rc == LDAP_OTHER ? "Internal error" : 751 mdb_strerror(rc), rc ); 752 Debug( LDAP_DEBUG_ANY, 753 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n", 754 text->bv_val, 0, 0 ); 755 e->e_id = NOID; 756 } 757 758 return e->e_id; 759 } 760 761 static int mdb_dn2id_upgrade( BackendDB *be ); 762 763 int mdb_tool_entry_reindex( 764 BackendDB *be, 765 ID id, 766 AttributeDescription **adv ) 767 { 768 struct mdb_info *mi = (struct mdb_info *) be->be_private; 769 int rc; 770 Entry *e; 771 Operation op = {0}; 772 Opheader ohdr = {0}; 773 774 Debug( LDAP_DEBUG_ARGS, 775 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n", 776 (long) id, 0, 0 ); 777 assert( tool_base == NULL ); 778 assert( tool_filter == NULL ); 779 780 /* Special: do a dn2id upgrade */ 781 if ( adv && adv[0] == slap_schema.si_ad_entryDN ) { 782 /* short-circuit tool_entry_next() */ 783 mdb_cursor_get( cursor, &key, &data, MDB_LAST ); 784 return mdb_dn2id_upgrade( be ); 785 } 786 787 /* No indexes configured, nothing to do. Could return an 788 * error here to shortcut things. 789 */ 790 if (!mi->mi_attrs) { 791 return 0; 792 } 793 794 /* Check for explicit list of attrs to index */ 795 if ( adv ) { 796 int i, j, n; 797 798 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) { 799 /* count */ 800 for ( n = 0; adv[n]; n++ ) ; 801 802 /* insertion sort */ 803 for ( i = 0; i < n; i++ ) { 804 AttributeDescription *ad = adv[i]; 805 for ( j = i-1; j>=0; j--) { 806 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break; 807 adv[j+1] = adv[j]; 808 } 809 adv[j+1] = ad; 810 } 811 } 812 813 for ( i = 0; adv[i]; i++ ) { 814 if ( mi->mi_attrs[i]->ai_desc != adv[i] ) { 815 for ( j = i+1; j < mi->mi_nattrs; j++ ) { 816 if ( mi->mi_attrs[j]->ai_desc == adv[i] ) { 817 AttrInfo *ai = mi->mi_attrs[i]; 818 mi->mi_attrs[i] = mi->mi_attrs[j]; 819 mi->mi_attrs[j] = ai; 820 break; 821 } 822 } 823 if ( j == mi->mi_nattrs ) { 824 Debug( LDAP_DEBUG_ANY, 825 LDAP_XSTRING(mdb_tool_entry_reindex) 826 ": no index configured for %s\n", 827 adv[i]->ad_cname.bv_val, 0, 0 ); 828 return -1; 829 } 830 } 831 } 832 mi->mi_nattrs = i; 833 } 834 835 e = mdb_tool_entry_get( be, id ); 836 837 if( e == NULL ) { 838 Debug( LDAP_DEBUG_ANY, 839 LDAP_XSTRING(mdb_tool_entry_reindex) 840 ": could not locate id=%ld\n", 841 (long) id, 0, 0 ); 842 return -1; 843 } 844 845 if ( !txi ) { 846 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi ); 847 if( rc != 0 ) { 848 Debug( LDAP_DEBUG_ANY, 849 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": " 850 "txn_begin failed: %s (%d)\n", 851 mdb_strerror(rc), rc, 0 ); 852 goto done; 853 } 854 } 855 856 if ( slapMode & SLAP_TRUNCATE_MODE ) { 857 int i; 858 for ( i=0; i < mi->mi_nattrs; i++ ) { 859 rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 ); 860 if ( rc ) { 861 Debug( LDAP_DEBUG_ANY, 862 LDAP_XSTRING(mdb_tool_entry_reindex) 863 ": (Truncate) mdb_drop(%s) failed: %s (%d)\n", 864 mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val, 865 mdb_strerror(rc), rc ); 866 return -1; 867 } 868 } 869 slapMode ^= SLAP_TRUNCATE_MODE; 870 } 871 872 /* 873 * just (re)add them for now 874 * Use truncate mode to empty/reset index databases 875 */ 876 877 Debug( LDAP_DEBUG_TRACE, 878 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n", 879 (long) id, 0, 0 ); 880 881 op.o_hdr = &ohdr; 882 op.o_bd = be; 883 op.o_tmpmemctx = NULL; 884 op.o_tmpmfuncs = &ch_mfuncs; 885 886 rc = mdb_tool_index_add( &op, txi, e ); 887 888 done: 889 if( rc == 0 ) { 890 mdb_writes++; 891 if ( mdb_writes >= mdb_writes_per_commit ) { 892 MDB_val key; 893 unsigned i; 894 MDB_TOOL_IDL_FLUSH( be, txi ); 895 rc = mdb_txn_commit( txi ); 896 mdb_writes = 0; 897 for ( i=0; i<mi->mi_nattrs; i++ ) 898 mi->mi_attrs[i]->ai_cursor = NULL; 899 if( rc != 0 ) { 900 Debug( LDAP_DEBUG_ANY, 901 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) 902 ": txn_commit failed: %s (%d)\n", 903 mdb_strerror(rc), rc, 0 ); 904 e->e_id = NOID; 905 } 906 mdb_cursor_close( cursor ); 907 txi = NULL; 908 /* Must close the read txn to allow old pages to be reclaimed. */ 909 mdb_txn_abort( mdb_tool_txn ); 910 /* and then reopen it so that tool_entry_next still works. */ 911 mdb_txn_begin( mi->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn ); 912 mdb_cursor_open( mdb_tool_txn, mi->mi_id2entry, &cursor ); 913 key.mv_data = &id; 914 key.mv_size = sizeof(ID); 915 mdb_cursor_get( cursor, &key, NULL, MDB_SET ); 916 } 917 918 } else { 919 unsigned i; 920 mdb_writes = 0; 921 mdb_cursor_close( cursor ); 922 cursor = NULL; 923 mdb_txn_abort( txi ); 924 for ( i=0; i<mi->mi_nattrs; i++ ) 925 mi->mi_attrs[i]->ai_cursor = NULL; 926 Debug( LDAP_DEBUG_ANY, 927 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) 928 ": txn_aborted! err=%d\n", 929 rc, 0, 0 ); 930 e->e_id = NOID; 931 txi = NULL; 932 } 933 mdb_entry_release( &op, e, 0 ); 934 935 return rc; 936 } 937 938 ID mdb_tool_entry_modify( 939 BackendDB *be, 940 Entry *e, 941 struct berval *text ) 942 { 943 int rc; 944 struct mdb_info *mdb; 945 Operation op = {0}; 946 Opheader ohdr = {0}; 947 948 assert( be != NULL ); 949 assert( slapMode & SLAP_TOOL_MODE ); 950 951 assert( text != NULL ); 952 assert( text->bv_val != NULL ); 953 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 954 955 assert ( e->e_id != NOID ); 956 957 Debug( LDAP_DEBUG_TRACE, 958 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n", 959 (long) e->e_id, e->e_dn, 0 ); 960 961 mdb = (struct mdb_info *) be->be_private; 962 963 if( cursor ) { 964 mdb_cursor_close( cursor ); 965 cursor = NULL; 966 } 967 if ( !mdb_tool_txn ) { 968 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn ); 969 if( rc != 0 ) { 970 snprintf( text->bv_val, text->bv_len, 971 "txn_begin failed: %s (%d)", 972 mdb_strerror(rc), rc ); 973 Debug( LDAP_DEBUG_ANY, 974 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n", 975 text->bv_val, 0, 0 ); 976 return NOID; 977 } 978 } 979 980 op.o_hdr = &ohdr; 981 op.o_bd = be; 982 op.o_tmpmemctx = NULL; 983 op.o_tmpmfuncs = &ch_mfuncs; 984 985 /* id2entry index */ 986 rc = mdb_id2entry_update( &op, mdb_tool_txn, NULL, e ); 987 if( rc != 0 ) { 988 snprintf( text->bv_val, text->bv_len, 989 "id2entry_update failed: err=%d", rc ); 990 Debug( LDAP_DEBUG_ANY, 991 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n", 992 text->bv_val, 0, 0 ); 993 goto done; 994 } 995 996 done: 997 if( rc == 0 ) { 998 rc = mdb_txn_commit( mdb_tool_txn ); 999 if( rc != 0 ) { 1000 mdb->mi_numads = 0; 1001 snprintf( text->bv_val, text->bv_len, 1002 "txn_commit failed: %s (%d)", 1003 mdb_strerror(rc), rc ); 1004 Debug( LDAP_DEBUG_ANY, 1005 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": " 1006 "%s\n", text->bv_val, 0, 0 ); 1007 e->e_id = NOID; 1008 } 1009 1010 } else { 1011 mdb_txn_abort( mdb_tool_txn ); 1012 snprintf( text->bv_val, text->bv_len, 1013 "txn_aborted! %s (%d)", 1014 mdb_strerror(rc), rc ); 1015 Debug( LDAP_DEBUG_ANY, 1016 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n", 1017 text->bv_val, 0, 0 ); 1018 e->e_id = NOID; 1019 } 1020 mdb_tool_txn = NULL; 1021 idcursor = NULL; 1022 1023 return e->e_id; 1024 } 1025 1026 static void * 1027 mdb_tool_index_task( void *ctx, void *ptr ) 1028 { 1029 int base = *(int *)ptr; 1030 1031 free( ptr ); 1032 while ( 1 ) { 1033 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex ); 1034 mdb_tool_index_tcount--; 1035 if ( !mdb_tool_index_tcount ) 1036 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main ); 1037 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work, 1038 &mdb_tool_index_mutex ); 1039 if ( slapd_shutdown ) { 1040 mdb_tool_index_tcount--; 1041 if ( !mdb_tool_index_tcount ) 1042 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main ); 1043 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 1044 break; 1045 } 1046 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex ); 1047 mdb_tool_index_rec[base].ir_i = mdb_index_recrun( mdb_tool_ix_op, 1048 mdb_tool_ix_txn, 1049 mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base ); 1050 } 1051 1052 return NULL; 1053 } 1054 1055 #ifdef MDB_TOOL_IDL_CACHING 1056 static int 1057 mdb_tool_idl_cmp( const void *v1, const void *v2 ) 1058 { 1059 const mdb_tool_idl_cache *c1 = v1, *c2 = v2; 1060 int rc; 1061 1062 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc; 1063 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len ); 1064 } 1065 1066 static int 1067 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic ) 1068 { 1069 mdb_tool_idl_cache_entry *ice; 1070 MDB_val key, data[2]; 1071 int i, rc; 1072 ID id, nid; 1073 1074 /* Freshly allocated, ignore it */ 1075 if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) { 1076 return 0; 1077 } 1078 1079 key.mv_data = ic->kstr.bv_val; 1080 key.mv_size = ic->kstr.bv_len; 1081 1082 if ( ic->count > MDB_IDL_DB_SIZE ) { 1083 while ( ic->flags & WAS_FOUND ) { 1084 rc = mdb_cursor_get( mc, &key, data, MDB_SET ); 1085 if ( rc ) { 1086 /* FIXME: find out why this happens */ 1087 ic->flags = 0; 1088 break; 1089 } 1090 if ( ic->flags & WAS_RANGE ) { 1091 /* Skip lo */ 1092 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); 1093 1094 /* Get hi */ 1095 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); 1096 1097 /* Store range hi */ 1098 data[0].mv_data = &ic->last; 1099 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT ); 1100 } else { 1101 /* Delete old data, replace with range */ 1102 ic->first = *(ID *)data[0].mv_data; 1103 mdb_cursor_del( mc, MDB_NODUPDATA ); 1104 } 1105 break; 1106 } 1107 if ( !(ic->flags & WAS_RANGE)) { 1108 /* range, didn't exist before */ 1109 nid = 0; 1110 data[0].mv_size = sizeof(ID); 1111 data[0].mv_data = &nid; 1112 rc = mdb_cursor_put( mc, &key, data, 0 ); 1113 if ( rc == 0 ) { 1114 data[0].mv_data = &ic->first; 1115 rc = mdb_cursor_put( mc, &key, data, 0 ); 1116 if ( rc == 0 ) { 1117 data[0].mv_data = &ic->last; 1118 rc = mdb_cursor_put( mc, &key, data, 0 ); 1119 } 1120 } 1121 if ( rc ) { 1122 rc = -1; 1123 } 1124 } 1125 } else { 1126 /* Normal write */ 1127 int n; 1128 1129 data[0].mv_size = sizeof(ID); 1130 rc = 0; 1131 i = ic->offset; 1132 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) { 1133 int end; 1134 if ( ice->next ) { 1135 end = IDBLOCK; 1136 } else { 1137 end = ic->count & (IDBLOCK-1); 1138 if ( !end ) 1139 end = IDBLOCK; 1140 } 1141 data[1].mv_size = end - i; 1142 data[0].mv_data = &ice->ids[i]; 1143 i = 0; 1144 rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE ); 1145 if ( rc ) { 1146 if ( rc == MDB_KEYEXIST ) { 1147 rc = 0; 1148 continue; 1149 } 1150 rc = -1; 1151 break; 1152 } 1153 } 1154 if ( ic->head ) { 1155 ic->tail->next = ai->ai_flist; 1156 ai->ai_flist = ic->head; 1157 } 1158 } 1159 ic->head = ai->ai_clist; 1160 ai->ai_clist = ic; 1161 return rc; 1162 } 1163 1164 static int 1165 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai ) 1166 { 1167 MDB_cursor *mc; 1168 Avlnode *root; 1169 int rc; 1170 1171 mdb_cursor_open( txn, ai->ai_dbi, &mc ); 1172 root = tavl_end( ai->ai_root, TAVL_DIR_LEFT ); 1173 do { 1174 rc = mdb_tool_idl_flush_one( mc, ai, root->avl_data ); 1175 if ( rc != -1 ) 1176 rc = 0; 1177 } while ((root = tavl_next(root, TAVL_DIR_RIGHT))); 1178 mdb_cursor_close( mc ); 1179 1180 return rc; 1181 } 1182 1183 static int 1184 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn ) 1185 { 1186 struct mdb_info *mdb = (struct mdb_info *) be->be_private; 1187 int rc = 0; 1188 unsigned int i, dbi; 1189 1190 for ( i=0; i < mdb->mi_nattrs; i++ ) { 1191 if ( !mdb->mi_attrs[i]->ai_root ) continue; 1192 rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i] ); 1193 tavl_free(mdb->mi_attrs[i]->ai_root, NULL); 1194 mdb->mi_attrs[i]->ai_root = NULL; 1195 if ( rc ) 1196 break; 1197 } 1198 return rc; 1199 } 1200 1201 int mdb_tool_idl_add( 1202 BackendDB *be, 1203 MDB_cursor *mc, 1204 struct berval *keys, 1205 ID id ) 1206 { 1207 MDB_dbi dbi; 1208 mdb_tool_idl_cache *ic, itmp; 1209 mdb_tool_idl_cache_entry *ice; 1210 int i, rc, lcount; 1211 AttrInfo *ai = (AttrInfo *)mc; 1212 mc = ai->ai_cursor; 1213 1214 dbi = ai->ai_dbi; 1215 for (i=0; keys[i].bv_val; i++) { 1216 itmp.kstr = keys[i]; 1217 ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp ); 1218 1219 /* No entry yet, create one */ 1220 if ( !ic ) { 1221 MDB_val key, data; 1222 ID nid; 1223 int rc; 1224 1225 if ( ai->ai_clist ) { 1226 ic = ai->ai_clist; 1227 ai->ai_clist = ic->head; 1228 } else { 1229 ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 ); 1230 } 1231 ic->kstr.bv_len = itmp.kstr.bv_len; 1232 ic->kstr.bv_val = (char *)(ic+1); 1233 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len ); 1234 ic->head = ic->tail = NULL; 1235 ic->last = 0; 1236 ic->count = 0; 1237 ic->offset = 0; 1238 ic->flags = 0; 1239 tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp, 1240 avl_dup_error ); 1241 1242 /* load existing key count here */ 1243 key.mv_size = keys[i].bv_len; 1244 key.mv_data = keys[i].bv_val; 1245 rc = mdb_cursor_get( mc, &key, &data, MDB_SET ); 1246 if ( rc == 0 ) { 1247 ic->flags |= WAS_FOUND; 1248 nid = *(ID *)data.mv_data; 1249 if ( nid == 0 ) { 1250 ic->count = MDB_IDL_DB_SIZE+1; 1251 ic->flags |= WAS_RANGE; 1252 } else { 1253 size_t count; 1254 1255 mdb_cursor_count( mc, &count ); 1256 ic->count = count; 1257 ic->first = nid; 1258 ic->offset = count & (IDBLOCK-1); 1259 } 1260 } 1261 } 1262 /* are we a range already? */ 1263 if ( ic->count > MDB_IDL_DB_SIZE ) { 1264 ic->last = id; 1265 continue; 1266 /* Are we at the limit, and converting to a range? */ 1267 } else if ( ic->count == MDB_IDL_DB_SIZE ) { 1268 if ( ic->head ) { 1269 ic->tail->next = ai->ai_flist; 1270 ai->ai_flist = ic->head; 1271 } 1272 ic->head = ic->tail = NULL; 1273 ic->last = id; 1274 ic->count++; 1275 continue; 1276 } 1277 /* No free block, create that too */ 1278 lcount = ic->count & (IDBLOCK-1); 1279 if ( !ic->tail || lcount == 0) { 1280 if ( ai->ai_flist ) { 1281 ice = ai->ai_flist; 1282 ai->ai_flist = ice->next; 1283 } else { 1284 ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry )); 1285 } 1286 ice->next = NULL; 1287 if ( !ic->head ) { 1288 ic->head = ice; 1289 } else { 1290 ic->tail->next = ice; 1291 } 1292 ic->tail = ice; 1293 if ( lcount ) 1294 ice->ids[lcount-1] = 0; 1295 if ( !ic->count ) 1296 ic->first = id; 1297 } 1298 ice = ic->tail; 1299 if (!lcount || ice->ids[lcount-1] != id) 1300 ice->ids[lcount] = id; 1301 ic->count++; 1302 } 1303 1304 return 0; 1305 } 1306 #endif /* MDB_TOOL_IDL_CACHING */ 1307 1308 /* Upgrade from pre 2.4.34 dn2id format */ 1309 1310 #include <ac/unistd.h> 1311 #include <lutil_meter.h> 1312 1313 #define STACKSIZ 2048 1314 1315 typedef struct rec { 1316 ID id; 1317 size_t len; 1318 char rdn[512]; 1319 } rec; 1320 1321 static int 1322 mdb_dn2id_upgrade( BackendDB *be ) { 1323 struct mdb_info *mi = (struct mdb_info *) be->be_private; 1324 MDB_txn *mt; 1325 MDB_cursor *mc = NULL; 1326 MDB_val key, data; 1327 char *ptr; 1328 int rc, writes=0, depth=0; 1329 int enable_meter = 0; 1330 ID id = 0, *num, count = 0; 1331 rec *stack; 1332 lutil_meter_t meter; 1333 1334 if (!(mi->mi_flags & MDB_NEED_UPGRADE)) { 1335 Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n", 1336 be->be_suffix[0].bv_val, 0, 0 ); 1337 return 0; 1338 } 1339 1340 { 1341 MDB_stat st; 1342 1343 mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st); 1344 if (!st.ms_entries) { 1345 /* Empty DB, nothing to upgrade? */ 1346 return 0; 1347 } 1348 if (isatty(2)) 1349 enable_meter = !lutil_meter_open(&meter, 1350 &lutil_meter_text_display, 1351 &lutil_meter_linear_estimator, 1352 st.ms_entries); 1353 } 1354 1355 num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec))); 1356 stack = (rec *)(num + STACKSIZ); 1357 1358 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); 1359 if (rc) { 1360 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n", 1361 mdb_strerror(rc), rc, 0 ); 1362 goto leave; 1363 } 1364 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); 1365 if (rc) { 1366 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n", 1367 mdb_strerror(rc), rc, 0 ); 1368 goto leave; 1369 } 1370 1371 key.mv_size = sizeof(ID); 1372 /* post-order depth-first update */ 1373 for(;;) { 1374 size_t dkids; 1375 unsigned char *ptr; 1376 1377 /* visit */ 1378 key.mv_data = &id; 1379 stack[depth].id = id; 1380 rc = mdb_cursor_get(mc, &key, &data, MDB_SET); 1381 if (rc) { 1382 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n", 1383 mdb_strerror(rc), rc, 0 ); 1384 goto leave; 1385 } 1386 num[depth] = 1; 1387 1388 rc = mdb_cursor_count(mc, &dkids); 1389 if (rc) { 1390 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n", 1391 mdb_strerror(rc), rc, 0 ); 1392 goto leave; 1393 } 1394 if (dkids > 1) { 1395 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); 1396 down: 1397 ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID); 1398 memcpy(&id, ptr, sizeof(ID)); 1399 depth++; 1400 memcpy(stack[depth].rdn, data.mv_data, data.mv_size); 1401 stack[depth].len = data.mv_size; 1402 continue; 1403 } 1404 1405 1406 /* pop: write updated count, advance to next node */ 1407 pop: 1408 /* update superior counts */ 1409 if (depth) 1410 num[depth-1] += num[depth]; 1411 1412 key.mv_data = &id; 1413 id = stack[depth-1].id; 1414 data.mv_data = stack[depth].rdn; 1415 data.mv_size = stack[depth].len; 1416 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); 1417 if (rc) { 1418 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n", 1419 mdb_strerror(rc), rc, 0 ); 1420 goto leave; 1421 } 1422 data.mv_data = stack[depth].rdn; 1423 ptr = (unsigned char *)data.mv_data + data.mv_size; 1424 memcpy(ptr, &num[depth], sizeof(ID)); 1425 data.mv_size += sizeof(ID); 1426 rc = mdb_cursor_del(mc, 0); 1427 if (rc) { 1428 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n", 1429 mdb_strerror(rc), rc, 0 ); 1430 goto leave; 1431 } 1432 rc = mdb_cursor_put(mc, &key, &data, 0); 1433 if (rc) { 1434 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n", 1435 mdb_strerror(rc), rc, 0 ); 1436 goto leave; 1437 } 1438 count++; 1439 #if 1 1440 if (enable_meter) 1441 lutil_meter_update(&meter, count, 0); 1442 #else 1443 { 1444 int len; 1445 ptr = data.mv_data; 1446 len = (ptr[0] & 0x7f) << 8 | ptr[1]; 1447 printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2); 1448 } 1449 #endif 1450 writes++; 1451 if (writes == 1000) { 1452 mdb_cursor_close(mc); 1453 rc = mdb_txn_commit(mt); 1454 if (rc) { 1455 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n", 1456 mdb_strerror(rc), rc, 0 ); 1457 goto leave; 1458 } 1459 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); 1460 if (rc) { 1461 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n", 1462 mdb_strerror(rc), rc, 0 ); 1463 goto leave; 1464 } 1465 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); 1466 if (rc) { 1467 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n", 1468 mdb_strerror(rc), rc, 0 ); 1469 goto leave; 1470 } 1471 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); 1472 if (rc) { 1473 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n", 1474 mdb_strerror(rc), rc, 0 ); 1475 goto leave; 1476 } 1477 writes = 0; 1478 } 1479 depth--; 1480 1481 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); 1482 if (rc == 0) 1483 goto down; 1484 rc = 0; 1485 if (depth) 1486 goto pop; 1487 else 1488 break; 1489 } 1490 leave: 1491 mdb_cursor_close(mc); 1492 if (mt) { 1493 int r2; 1494 r2 = mdb_txn_commit(mt); 1495 if (r2) { 1496 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n", 1497 mdb_strerror(r2), r2, 0 ); 1498 if (!rc) 1499 rc = r2; 1500 } 1501 } 1502 ch_free(num); 1503 if (enable_meter) { 1504 lutil_meter_update(&meter, count, 1); 1505 lutil_meter_close(&meter); 1506 } 1507 return rc; 1508 } 1509