xref: /netbsd-src/external/bsd/openldap/dist/servers/slapd/back-mdb/tools.c (revision 549b59ed3ccf0d36d3097190a0db27b770f3a839)
1 /*	$NetBSD: tools.c,v 1.3 2021/08/14 16:15:00 christos Exp $	*/
2 
3 /* tools.c - tools for slap tools */
4 /* $OpenLDAP$ */
5 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
6  *
7  * Copyright 2011-2021 The OpenLDAP Foundation.
8  * All rights reserved.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted only as authorized by the OpenLDAP
12  * Public License.
13  *
14  * A copy of this license is available in the file LICENSE in the
15  * top-level directory of the distribution or, alternatively, at
16  * <http://www.OpenLDAP.org/license.html>.
17  */
18 
19 #include <sys/cdefs.h>
20 __RCSID("$NetBSD: tools.c,v 1.3 2021/08/14 16:15:00 christos Exp $");
21 
22 #include "portable.h"
23 
24 #include <stdio.h>
25 #include <ac/string.h>
26 #include <ac/errno.h>
27 
28 #define AVL_INTERNAL
29 #include "back-mdb.h"
30 #include "idl.h"
31 
32 #ifdef MDB_TOOL_IDL_CACHING
33 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
34 
35 #define	IDBLOCK	1024
36 
37 typedef struct mdb_tool_idl_cache_entry {
38 	struct mdb_tool_idl_cache_entry *next;
39 	ID ids[IDBLOCK];
40 } mdb_tool_idl_cache_entry;
41 
42 typedef struct mdb_tool_idl_cache {
43 	struct berval kstr;
44 	mdb_tool_idl_cache_entry *head, *tail;
45 	ID first, last;
46 	int count;
47 	short offset;
48 	short flags;
49 } mdb_tool_idl_cache;
50 #define WAS_FOUND	0x01
51 #define WAS_RANGE	0x02
52 
53 #define MDB_TOOL_IDL_FLUSH(be, txn)	mdb_tool_idl_flush(be, txn)
54 #else
55 #define MDB_TOOL_IDL_FLUSH(be, txn)
56 #endif /* MDB_TOOL_IDL_CACHING */
57 
58 MDB_txn *mdb_tool_txn = NULL;
59 
60 static MDB_txn *txi = NULL;
61 static MDB_cursor *cursor = NULL, *idcursor = NULL;
62 static MDB_cursor *mcp = NULL, *mcd = NULL;
63 static MDB_val key, data;
64 static ID previd = NOID;
65 
66 typedef struct dn_id {
67 	ID id;
68 	struct berval dn;
69 } dn_id;
70 
71 #define	HOLE_SIZE	4096
72 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
73 static unsigned nhmax = HOLE_SIZE;
74 static unsigned nholes;
75 
76 static struct berval	*tool_base;
77 static int		tool_scope;
78 static Filter		*tool_filter;
79 static Entry		*tool_next_entry;
80 
81 static ID mdb_tool_ix_id;
82 static BackendDB *mdb_tool_ix_be;
83 static MDB_txn *mdb_tool_ix_txn;
84 static int mdb_tool_index_tcount, mdb_tool_threads;
85 static IndexRec *mdb_tool_index_rec;
86 static AttrIxInfo **mdb_tool_axinfo;
87 static struct mdb_info *mdb_tool_info;
88 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
89 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
90 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
91 static void * mdb_tool_index_task( void *ctx, void *ptr );
92 
93 static int	mdb_writes, mdb_writes_per_commit;
94 
95 /* Number of ops per commit in Quick mode.
96  * Batching speeds writes overall, but too large a
97  * batch will fail with MDB_TXN_FULL.
98  */
99 #ifndef MDB_WRITES_PER_COMMIT
100 #define MDB_WRITES_PER_COMMIT	500
101 #endif
102 
103 static int
104 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
105 
mdb_tool_entry_open(BackendDB * be,int mode)106 int mdb_tool_entry_open(
107 	BackendDB *be, int mode )
108 {
109 	/* In Quick mode, commit once per 500 entries */
110 	mdb_writes = 0;
111 	if ( slapMode & SLAP_TOOL_QUICK )
112 		mdb_writes_per_commit = MDB_WRITES_PER_COMMIT;
113 	else
114 		mdb_writes_per_commit = 1;
115 
116 #ifdef MDB_TOOL_IDL_CACHING			/* threaded indexing has no performance advantage */
117 	/* Set up for threaded slapindex */
118 	if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
119 		if ( !mdb_tool_info ) {
120 			struct mdb_info *mdb = (struct mdb_info *) be->be_private;
121 			ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
122 			ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
123 			ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
124 			if ( mdb->mi_nattrs ) {
125 				int i;
126 				mdb_tool_threads = slap_tool_thread_max - 1;
127 				if ( mdb_tool_threads > 1 ) {
128 					mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec ));
129 					mdb_tool_axinfo = ch_calloc( mdb_tool_threads, sizeof( AttrIxInfo* ) +
130 						sizeof( AttrIxInfo ));
131 					mdb_tool_axinfo[0] = (AttrIxInfo *)(mdb_tool_axinfo + mdb_tool_threads);
132 					for (i=1; i<mdb_tool_threads; i++)
133 						mdb_tool_axinfo[i] = mdb_tool_axinfo[i-1]+1;
134 					mdb_tool_index_tcount = mdb_tool_threads - 1;
135 					mdb_tool_ix_be = be;
136 					for (i=1; i<mdb_tool_threads; i++) {
137 						int *ptr = ch_malloc( sizeof( int ));
138 						*ptr = i;
139 						ldap_pvt_thread_pool_submit( &connection_pool,
140 							mdb_tool_index_task, ptr );
141 					}
142 					mdb_tool_info = mdb;
143 				}
144 			}
145 		}
146 	}
147 #endif
148 
149 	return 0;
150 }
151 
mdb_tool_entry_close(BackendDB * be)152 int mdb_tool_entry_close(
153 	BackendDB *be )
154 {
155 #ifdef MDB_TOOL_IDL_CACHING
156 	if ( mdb_tool_info ) {
157 		int i;
158 		slapd_shutdown = 1;
159 		ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
160 
161 		/* There might still be some threads starting */
162 		while ( mdb_tool_index_tcount > 0 ) {
163 			ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
164 					&mdb_tool_index_mutex );
165 		}
166 
167 		mdb_tool_index_tcount = mdb_tool_threads - 1;
168 		ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
169 
170 		/* Make sure all threads are stopped */
171 		while ( mdb_tool_index_tcount > 0 ) {
172 			ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
173 				&mdb_tool_index_mutex );
174 		}
175 		ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
176 
177 		mdb_tool_info = NULL;
178 		slapd_shutdown = 0;
179 		ch_free( mdb_tool_index_rec );
180 		mdb_tool_index_tcount = mdb_tool_threads - 1;
181 		if (mdb_tool_txn)
182 			MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
183 		for (i=0; i<mdb_tool_threads; i++) {
184 			mdb_tool_idl_cache *ic;
185 			mdb_tool_idl_cache_entry *ice;
186 			while ((ic = mdb_tool_axinfo[i]->ai_clist)) {
187 				mdb_tool_axinfo[i]->ai_clist = ic->head;
188 				free(ic);
189 			}
190 			while ((ice = mdb_tool_axinfo[i]->ai_flist)) {
191 				mdb_tool_axinfo[i]->ai_flist = ice->next;
192 				free(ice);
193 			}
194 		}
195 	}
196 #endif
197 
198 	if( idcursor ) {
199 		mdb_cursor_close( idcursor );
200 		idcursor = NULL;
201 	}
202 	if( cursor ) {
203 		mdb_cursor_close( cursor );
204 		cursor = NULL;
205 	}
206 	{
207 		struct mdb_info *mdb = be->be_private;
208 		if ( mdb ) {
209 			int i;
210 			for (i=0; i<mdb->mi_nattrs; i++)
211 				mdb->mi_attrs[i]->ai_cursor = NULL;
212 		}
213 	}
214 	if( mdb_tool_txn ) {
215 		int rc;
216 		if (( rc = mdb_txn_commit( mdb_tool_txn ))) {
217 			Debug( LDAP_DEBUG_ANY,
218 				LDAP_XSTRING(mdb_tool_entry_close) ": database %s: "
219 				"txn_commit failed: %s (%d)\n",
220 				be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
221 			return -1;
222 		}
223 		mdb_tool_txn = NULL;
224 	}
225 	if( txi ) {
226 		int rc;
227 		if (( rc = mdb_txn_commit( txi ))) {
228 			Debug( LDAP_DEBUG_ANY,
229 				LDAP_XSTRING(mdb_tool_entry_close) ": database %s: "
230 				"txn_commit failed: %s (%d)\n",
231 				be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
232 			return -1;
233 		}
234 		txi = NULL;
235 	}
236 
237 	if( nholes ) {
238 		unsigned i;
239 		fprintf( stderr, "Error, entries missing!\n");
240 		for (i=0; i<nholes; i++) {
241 			fprintf(stderr, "  entry %ld: %s\n",
242 				holes[i].id, holes[i].dn.bv_val);
243 		}
244 		nholes = 0;
245 		return -1;
246 	}
247 
248 	return 0;
249 }
250 
251 ID
mdb_tool_entry_first_x(BackendDB * be,struct berval * base,int scope,Filter * f)252 mdb_tool_entry_first_x(
253 	BackendDB *be,
254 	struct berval *base,
255 	int scope,
256 	Filter *f )
257 {
258 	tool_base = base;
259 	tool_scope = scope;
260 	tool_filter = f;
261 
262 	return mdb_tool_entry_next( be );
263 }
264 
mdb_tool_entry_next(BackendDB * be)265 ID mdb_tool_entry_next(
266 	BackendDB *be )
267 {
268 	int rc;
269 	ID id;
270 	struct mdb_info *mdb;
271 
272 	assert( be != NULL );
273 	assert( slapMode & SLAP_TOOL_MODE );
274 
275 	mdb = (struct mdb_info *) be->be_private;
276 	assert( mdb != NULL );
277 
278 	if ( !mdb_tool_txn ) {
279 		rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
280 		if ( rc )
281 			return NOID;
282 		rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
283 		if ( rc ) {
284 			mdb_txn_abort( mdb_tool_txn );
285 			return NOID;
286 		}
287 	}
288 
289 next:;
290 	rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
291 
292 	if( rc ) {
293 		return NOID;
294 	}
295 
296 	previd = *(ID *)key.mv_data;
297 	id = previd;
298 
299 	if ( !data.mv_size )
300 		goto next;
301 
302 	if ( tool_filter || tool_base ) {
303 		static Operation op = {0};
304 		static Opheader ohdr = {0};
305 
306 		op.o_hdr = &ohdr;
307 		op.o_bd = be;
308 		op.o_tmpmemctx = NULL;
309 		op.o_tmpmfuncs = &ch_mfuncs;
310 
311 		if ( tool_next_entry ) {
312 			mdb_entry_release( &op, tool_next_entry, 0 );
313 			tool_next_entry = NULL;
314 		}
315 
316 		rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
317 		if ( rc == LDAP_NO_SUCH_OBJECT ) {
318 			goto next;
319 		}
320 
321 		assert( tool_next_entry != NULL );
322 
323 		if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
324 		{
325 			mdb_entry_release( &op, tool_next_entry, 0 );
326 			tool_next_entry = NULL;
327 			goto next;
328 		}
329 	}
330 
331 	return id;
332 }
333 
mdb_tool_dn2id_get(Backend * be,struct berval * dn)334 ID mdb_tool_dn2id_get(
335 	Backend *be,
336 	struct berval *dn
337 )
338 {
339 	struct mdb_info *mdb;
340 	Operation op = {0};
341 	Opheader ohdr = {0};
342 	ID id;
343 	int rc;
344 
345 	if ( BER_BVISEMPTY(dn) )
346 		return 0;
347 
348 	mdb = (struct mdb_info *) be->be_private;
349 
350 	if ( !mdb_tool_txn ) {
351 		rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
352 			MDB_RDONLY : 0, &mdb_tool_txn );
353 		if ( rc )
354 			return NOID;
355 	}
356 
357 	op.o_hdr = &ohdr;
358 	op.o_bd = be;
359 	op.o_tmpmemctx = NULL;
360 	op.o_tmpmfuncs = &ch_mfuncs;
361 
362 	rc = mdb_dn2id( &op, mdb_tool_txn, NULL, dn, &id, NULL, NULL, NULL );
363 	if ( rc == MDB_NOTFOUND )
364 		return NOID;
365 
366 	return id;
367 }
368 
369 static int
mdb_tool_entry_get_int(BackendDB * be,ID id,Entry ** ep)370 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
371 {
372 	Operation op = {0};
373 	Opheader ohdr = {0};
374 
375 	Entry *e = NULL;
376 	struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
377 	int rc;
378 
379 	assert( be != NULL );
380 	assert( slapMode & SLAP_TOOL_MODE );
381 
382 	if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
383 		*ep = tool_next_entry;
384 		tool_next_entry = NULL;
385 		return LDAP_SUCCESS;
386 	}
387 
388 	if ( id != previd ) {
389 		key.mv_size = sizeof(ID);
390 		key.mv_data = &id;
391 		rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
392 		if ( rc ) {
393 			rc = LDAP_OTHER;
394 			goto done;
395 		}
396 	}
397 	if ( !data.mv_size ) {
398 		rc = LDAP_NO_SUCH_OBJECT;
399 		goto done;
400 	}
401 
402 	op.o_hdr = &ohdr;
403 	op.o_bd = be;
404 	op.o_tmpmemctx = NULL;
405 	op.o_tmpmfuncs = &ch_mfuncs;
406 	if ( slapMode & SLAP_TOOL_READONLY ) {
407 		rc = mdb_id2name( &op, mdb_tool_txn, &idcursor, id, &dn, &ndn );
408 		if ( rc  ) {
409 			rc = LDAP_OTHER;
410 			goto done;
411 		}
412 		if ( tool_base != NULL ) {
413 			if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
414 				ch_free( dn.bv_val );
415 				ch_free( ndn.bv_val );
416 				rc = LDAP_NO_SUCH_OBJECT;
417 				goto done;
418 			}
419 		}
420 	}
421 	rc = mdb_entry_decode( &op, mdb_tool_txn, &data, id, &e );
422 	e->e_id = id;
423 	if ( !BER_BVISNULL( &dn )) {
424 		e->e_name = dn;
425 		e->e_nname = ndn;
426 	} else {
427 		e->e_name.bv_val = NULL;
428 		e->e_nname.bv_val = NULL;
429 	}
430 
431 done:
432 	if ( e != NULL ) {
433 		*ep = e;
434 	}
435 
436 	return rc;
437 }
438 
439 Entry*
mdb_tool_entry_get(BackendDB * be,ID id)440 mdb_tool_entry_get( BackendDB *be, ID id )
441 {
442 	Entry *e = NULL;
443 	int rc;
444 
445 	if ( !mdb_tool_txn ) {
446 		struct mdb_info *mdb = (struct mdb_info *) be->be_private;
447 		rc = mdb_txn_begin( mdb->mi_dbenv, NULL,
448 			(slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &mdb_tool_txn );
449 		if ( rc )
450 			return NULL;
451 	}
452 	if ( !cursor ) {
453 		struct mdb_info *mdb = (struct mdb_info *) be->be_private;
454 		rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
455 		if ( rc ) {
456 			mdb_txn_abort( mdb_tool_txn );
457 			mdb_tool_txn = NULL;
458 			return NULL;
459 		}
460 	}
461 	(void)mdb_tool_entry_get_int( be, id, &e );
462 	return e;
463 }
464 
mdb_tool_next_id(Operation * op,MDB_txn * tid,Entry * e,struct berval * text,int hole)465 static int mdb_tool_next_id(
466 	Operation *op,
467 	MDB_txn *tid,
468 	Entry *e,
469 	struct berval *text,
470 	int hole )
471 {
472 	struct berval dn = e->e_name;
473 	struct berval ndn = e->e_nname;
474 	struct berval pdn, npdn, nmatched;
475 	ID id, pid = 0;
476 	int rc;
477 
478 	if (ndn.bv_len == 0) {
479 		e->e_id = 0;
480 		return 0;
481 	}
482 
483 	rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, NULL, &nmatched );
484 	if ( rc == MDB_NOTFOUND ) {
485 		if ( !be_issuffix( op->o_bd, &ndn ) ) {
486 			ID eid = e->e_id;
487 			dnParent( &ndn, &npdn );
488 			if ( nmatched.bv_len != npdn.bv_len ) {
489 				dnParent( &dn, &pdn );
490 				e->e_name = pdn;
491 				e->e_nname = npdn;
492 				rc = mdb_tool_next_id( op, tid, e, text, 1 );
493 				e->e_name = dn;
494 				e->e_nname = ndn;
495 				if ( rc ) {
496 					return rc;
497 				}
498 				/* If parent didn't exist, it was created just now
499 				 * and its ID is now in e->e_id. Make sure the current
500 				 * entry gets added under the new parent ID.
501 				 */
502 				if ( eid != e->e_id ) {
503 					pid = e->e_id;
504 				}
505 			} else {
506 				pid = id;
507 			}
508 		}
509 		rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
510 		if ( rc ) {
511 			snprintf( text->bv_val, text->bv_len,
512 				"next_id failed: %s (%d)",
513 				mdb_strerror(rc), rc );
514 		Debug( LDAP_DEBUG_ANY,
515 			"=> mdb_tool_next_id: %s\n", text->bv_val );
516 			return rc;
517 		}
518 		rc = mdb_dn2id_add( op, mcp, mcd, pid, 1, 1, e );
519 		if ( rc ) {
520 			snprintf( text->bv_val, text->bv_len,
521 				"dn2id_add failed: %s (%d)",
522 				mdb_strerror(rc), rc );
523 			Debug( LDAP_DEBUG_ANY,
524 				"=> mdb_tool_next_id: %s\n", text->bv_val );
525 		} else if ( hole ) {
526 			MDB_val key, data;
527 			if ( nholes == nhmax - 1 ) {
528 				if ( holes == hbuf ) {
529 					holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
530 					AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
531 				} else {
532 					holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
533 				}
534 				nhmax *= 2;
535 			}
536 			ber_dupbv( &holes[nholes].dn, &ndn );
537 			holes[nholes++].id = e->e_id;
538 			key.mv_size = sizeof(ID);
539 			key.mv_data = &e->e_id;
540 			data.mv_size = 0;
541 			data.mv_data = NULL;
542 			rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
543 			if ( rc == MDB_KEYEXIST )
544 				rc = 0;
545 			if ( rc ) {
546 				snprintf( text->bv_val, text->bv_len,
547 					"dummy id2entry add failed: %s (%d)",
548 					mdb_strerror(rc), rc );
549 				Debug( LDAP_DEBUG_ANY,
550 					"=> mdb_tool_next_id: %s\n", text->bv_val );
551 			}
552 		}
553 	} else if ( !hole ) {
554 		unsigned i, j;
555 
556 		e->e_id = id;
557 
558 		for ( i=0; i<nholes; i++) {
559 			if ( holes[i].id == e->e_id ) {
560 				free(holes[i].dn.bv_val);
561 				for (j=i;j<nholes;j++) holes[j] = holes[j+1];
562 				holes[j].id = 0;
563 				nholes--;
564 				break;
565 			} else if ( holes[i].id > e->e_id ) {
566 				break;
567 			}
568 		}
569 	}
570 	return rc;
571 }
572 
573 static int
mdb_tool_index_add(Operation * op,MDB_txn * txn,Entry * e)574 mdb_tool_index_add(
575 	Operation *op,
576 	MDB_txn *txn,
577 	Entry *e )
578 {
579 	struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
580 
581 	if ( !mdb->mi_nattrs )
582 		return 0;
583 
584 	if ( mdb_tool_threads > 1 ) {
585 		IndexRec *ir;
586 		int i, rc;
587 		Attribute *a;
588 
589 		ir = mdb_tool_index_rec;
590 		for (i=0; i<mdb->mi_nattrs; i++)
591 			ir[i].ir_attrs = NULL;
592 
593 		for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
594 			rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
595 				&a->a_desc->ad_tags, ir );
596 			if ( rc )
597 				return rc;
598 		}
599 		for (i=0; i<mdb->mi_nattrs; i++) {
600 			if ( !ir[i].ir_ai )
601 				break;
602 			rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi,
603 				 &ir[i].ir_ai->ai_cursor );
604 			if ( rc )
605 				return rc;
606 		}
607 		mdb_tool_ix_id = e->e_id;
608 		mdb_tool_ix_txn = txn;
609 		ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
610 		/* Wait for all threads to be ready */
611 		while ( mdb_tool_index_tcount ) {
612 			ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
613 				&mdb_tool_index_mutex );
614 		}
615 
616 		for ( i=1; i<mdb_tool_threads; i++ )
617 			mdb_tool_index_rec[i].ir_i = LDAP_BUSY;
618 		mdb_tool_index_tcount = mdb_tool_threads - 1;
619 		ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
620 		ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
621 
622 		return mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 );
623 	} else
624 	{
625 		return mdb_index_entry_add( op, txn, e );
626 	}
627 }
628 
629 static int
mdb_tool_index_finish()630 mdb_tool_index_finish()
631 {
632 	int i, rc = 0;
633 	ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
634 	for ( i=1; i<mdb_tool_threads; i++ ) {
635 		if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) {
636 			ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
637 				&mdb_tool_index_mutex );
638 			i--;
639 			continue;
640 		}
641 		if ( mdb_tool_index_rec[i].ir_i ) {
642 			rc = mdb_tool_index_rec[i].ir_i;
643 			break;
644 		}
645 	}
646 	ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
647 	return rc;
648 }
649 
mdb_tool_entry_put(BackendDB * be,Entry * e,struct berval * text)650 ID mdb_tool_entry_put(
651 	BackendDB *be,
652 	Entry *e,
653 	struct berval *text )
654 {
655 	int rc;
656 	struct mdb_info *mdb;
657 	Operation op = {0};
658 	Opheader ohdr = {0};
659 
660 	assert( be != NULL );
661 	assert( slapMode & SLAP_TOOL_MODE );
662 
663 	assert( text != NULL );
664 	assert( text->bv_val != NULL );
665 	assert( text->bv_val[0] == '\0' );	/* overconservative? */
666 
667 	Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
668 		"( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn );
669 
670 	mdb = (struct mdb_info *) be->be_private;
671 
672 	if ( !mdb_tool_txn ) {
673 		rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
674 		if( rc != 0 ) {
675 			snprintf( text->bv_val, text->bv_len,
676 				"txn_begin failed: %s (%d)",
677 				mdb_strerror(rc), rc );
678 			Debug( LDAP_DEBUG_ANY,
679 				"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
680 				 text->bv_val );
681 			return NOID;
682 		}
683 	}
684 	if ( !idcursor ) {
685 		rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &idcursor );
686 		if( rc != 0 ) {
687 			snprintf( text->bv_val, text->bv_len,
688 				"cursor_open failed: %s (%d)",
689 				mdb_strerror(rc), rc );
690 			Debug( LDAP_DEBUG_ANY,
691 				"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
692 				 text->bv_val );
693 			return NOID;
694 		}
695 		if ( !mdb->mi_nextid ) {
696 			ID dummy;
697 			mdb_next_id( be, idcursor, &dummy );
698 		}
699 		rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcp );
700 		if( rc != 0 ) {
701 			snprintf( text->bv_val, text->bv_len,
702 				"cursor_open failed: %s (%d)",
703 				mdb_strerror(rc), rc );
704 			Debug( LDAP_DEBUG_ANY,
705 				"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
706 				 text->bv_val );
707 			return NOID;
708 		}
709 		rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcd );
710 		if( rc != 0 ) {
711 			snprintf( text->bv_val, text->bv_len,
712 				"cursor_open failed: %s (%d)",
713 				mdb_strerror(rc), rc );
714 			Debug( LDAP_DEBUG_ANY,
715 				"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
716 				 text->bv_val );
717 			return NOID;
718 		}
719 	}
720 
721 	op.o_hdr = &ohdr;
722 	op.o_bd = be;
723 	op.o_tmpmemctx = NULL;
724 	op.o_tmpmfuncs = &ch_mfuncs;
725 
726 	/* add dn2id indices */
727 	rc = mdb_tool_next_id( &op, mdb_tool_txn, e, text, 0 );
728 	if( rc != 0 ) {
729 		goto done;
730 	}
731 
732 	if ( mdb_tool_threads > 1 ) {
733 		LDAP_SLIST_INSERT_HEAD( &op.o_extra, &mdb_tool_axinfo[0]->ai_oe, oe_next );
734 	}
735 	rc = mdb_tool_index_add( &op, mdb_tool_txn, e );
736 	if( rc != 0 ) {
737 		snprintf( text->bv_val, text->bv_len,
738 				"index_entry_add failed: err=%d", rc );
739 		Debug( LDAP_DEBUG_ANY,
740 			"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
741 			text->bv_val );
742 		goto done;
743 	}
744 
745 
746 	/* id2entry index */
747 	rc = mdb_id2entry_add( &op, mdb_tool_txn, idcursor, e );
748 	if( rc != 0 ) {
749 		snprintf( text->bv_val, text->bv_len,
750 				"id2entry_add failed: err=%d", rc );
751 		Debug( LDAP_DEBUG_ANY,
752 			"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
753 			text->bv_val );
754 		goto done;
755 	}
756 
757 	if( mdb->mi_nattrs && mdb_tool_threads > 1 )
758 		rc = mdb_tool_index_finish();
759 
760 done:
761 	if( rc == 0 ) {
762 		mdb_writes++;
763 		if ( mdb_writes >= mdb_writes_per_commit ) {
764 			unsigned i;
765 			MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
766 			rc = mdb_txn_commit( mdb_tool_txn );
767 			for ( i=0; i<mdb->mi_nattrs; i++ )
768 				mdb->mi_attrs[i]->ai_cursor = NULL;
769 			mdb_writes = 0;
770 			mdb_tool_txn = NULL;
771 			idcursor = NULL;
772 			if( rc != 0 ) {
773 				mdb->mi_numads = 0;
774 				snprintf( text->bv_val, text->bv_len,
775 						"txn_commit failed: %s (%d)",
776 						mdb_strerror(rc), rc );
777 				Debug( LDAP_DEBUG_ANY,
778 					"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
779 					text->bv_val );
780 				e->e_id = NOID;
781 			}
782 		}
783 
784 	} else {
785 		unsigned i;
786 		mdb_txn_abort( mdb_tool_txn );
787 		mdb_tool_txn = NULL;
788 		idcursor = NULL;
789 		for ( i=0; i<mdb->mi_nattrs; i++ )
790 			mdb->mi_attrs[i]->ai_cursor = NULL;
791 		mdb_writes = 0;
792 		snprintf( text->bv_val, text->bv_len,
793 			"txn_aborted! %s (%d)",
794 			rc == LDAP_OTHER ? "Internal error" :
795 			mdb_strerror(rc), rc );
796 		Debug( LDAP_DEBUG_ANY,
797 			"=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
798 			text->bv_val );
799 		e->e_id = NOID;
800 	}
801 
802 	return e->e_id;
803 }
804 
805 static int mdb_dn2id_upgrade( BackendDB *be );
806 
mdb_tool_entry_reindex(BackendDB * be,ID id,AttributeDescription ** adv)807 int mdb_tool_entry_reindex(
808 	BackendDB *be,
809 	ID id,
810 	AttributeDescription **adv )
811 {
812 	struct mdb_info *mi = (struct mdb_info *) be->be_private;
813 	int rc;
814 	Entry *e;
815 	Operation op = {0};
816 	Opheader ohdr = {0};
817 
818 	Debug( LDAP_DEBUG_ARGS,
819 		"=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
820 		(long) id );
821 	assert( tool_base == NULL );
822 	assert( tool_filter == NULL );
823 
824 	/* Special: do a dn2id upgrade */
825 	if ( adv && adv[0] == slap_schema.si_ad_entryDN ) {
826 		/* short-circuit tool_entry_next() */
827 		mdb_cursor_get( cursor, &key, &data, MDB_LAST );
828 		return mdb_dn2id_upgrade( be );
829 	}
830 
831 	/* No indexes configured, nothing to do. Could return an
832 	 * error here to shortcut things.
833 	 */
834 	if (!mi->mi_attrs) {
835 		return 0;
836 	}
837 
838 	/* Check for explicit list of attrs to index */
839 	if ( adv ) {
840 		int i, j, n;
841 
842 		if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
843 			/* count */
844 			for ( n = 0; adv[n]; n++ ) ;
845 
846 			/* insertion sort */
847 			for ( i = 0; i < n; i++ ) {
848 				AttributeDescription *ad = adv[i];
849 				for ( j = i-1; j>=0; j--) {
850 					if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
851 					adv[j+1] = adv[j];
852 				}
853 				adv[j+1] = ad;
854 			}
855 		}
856 
857 		for ( i = 0; adv[i]; i++ ) {
858 			if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
859 				for ( j = i+1; j < mi->mi_nattrs; j++ ) {
860 					if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
861 						AttrInfo *ai = mi->mi_attrs[i];
862 						mi->mi_attrs[i] = mi->mi_attrs[j];
863 						mi->mi_attrs[j] = ai;
864 						break;
865 					}
866 				}
867 				if ( j == mi->mi_nattrs ) {
868 					Debug( LDAP_DEBUG_ANY,
869 						LDAP_XSTRING(mdb_tool_entry_reindex)
870 						": no index configured for %s\n",
871 						adv[i]->ad_cname.bv_val );
872 					return -1;
873 				}
874 			}
875 		}
876 		mi->mi_nattrs = i;
877 	}
878 
879 	e = mdb_tool_entry_get( be, id );
880 
881 	if( e == NULL ) {
882 		Debug( LDAP_DEBUG_ANY,
883 			LDAP_XSTRING(mdb_tool_entry_reindex)
884 			": could not locate id=%ld\n",
885 			(long) id );
886 		return -1;
887 	}
888 
889 	if ( !txi ) {
890 		rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
891 		if( rc != 0 ) {
892 			Debug( LDAP_DEBUG_ANY,
893 				"=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
894 				"txn_begin failed: %s (%d)\n",
895 				mdb_strerror(rc), rc );
896 			goto done;
897 		}
898 	}
899 
900 	if ( slapMode & SLAP_TRUNCATE_MODE ) {
901 		int i;
902 		for ( i=0; i < mi->mi_nattrs; i++ ) {
903 			rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 );
904 			if ( rc ) {
905 				Debug( LDAP_DEBUG_ANY,
906 					LDAP_XSTRING(mdb_tool_entry_reindex)
907 					": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
908 					mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
909 					mdb_strerror(rc), rc );
910 				return -1;
911 			}
912 		}
913 		slapMode ^= SLAP_TRUNCATE_MODE;
914 	}
915 
916 	/*
917 	 * just (re)add them for now
918 	 * Use truncate mode to empty/reset index databases
919 	 */
920 
921 	Debug( LDAP_DEBUG_TRACE,
922 		"=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
923 		(long) id );
924 
925 	op.o_hdr = &ohdr;
926 	op.o_bd = be;
927 	op.o_tmpmemctx = NULL;
928 	op.o_tmpmfuncs = &ch_mfuncs;
929 
930 	rc = mdb_tool_index_add( &op, txi, e );
931 
932 done:
933 	if( rc == 0 ) {
934 		mdb_writes++;
935 		if ( mdb_writes >= mdb_writes_per_commit ) {
936 			MDB_val key;
937 			unsigned i;
938 			MDB_TOOL_IDL_FLUSH( be, txi );
939 			rc = mdb_txn_commit( txi );
940 			mdb_writes = 0;
941 			for ( i=0; i<mi->mi_nattrs; i++ )
942 				mi->mi_attrs[i]->ai_cursor = NULL;
943 			if( rc != 0 ) {
944 				Debug( LDAP_DEBUG_ANY,
945 					"=> " LDAP_XSTRING(mdb_tool_entry_reindex)
946 					": txn_commit failed: %s (%d)\n",
947 					mdb_strerror(rc), rc );
948 				e->e_id = NOID;
949 			}
950 			mdb_cursor_close( cursor );
951 			txi = NULL;
952 			/* Must close the read txn to allow old pages to be reclaimed. */
953 			mdb_txn_abort( mdb_tool_txn );
954 			/* and then reopen it so that tool_entry_next still works. */
955 			mdb_txn_begin( mi->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
956 			mdb_cursor_open( mdb_tool_txn, mi->mi_id2entry, &cursor );
957 			key.mv_data = &id;
958 			key.mv_size = sizeof(ID);
959 			mdb_cursor_get( cursor, &key, NULL, MDB_SET );
960 		}
961 
962 	} else {
963 		unsigned i;
964 		mdb_writes = 0;
965 		mdb_cursor_close( cursor );
966 		cursor = NULL;
967 		mdb_txn_abort( txi );
968 		for ( i=0; i<mi->mi_nattrs; i++ )
969 			mi->mi_attrs[i]->ai_cursor = NULL;
970 		Debug( LDAP_DEBUG_ANY,
971 			"=> " LDAP_XSTRING(mdb_tool_entry_reindex)
972 			": txn_aborted! err=%d\n",
973 			rc );
974 		e->e_id = NOID;
975 		txi = NULL;
976 	}
977 	mdb_entry_release( &op, e, 0 );
978 
979 	return rc;
980 }
981 
mdb_tool_entry_modify(BackendDB * be,Entry * e,struct berval * text)982 ID mdb_tool_entry_modify(
983 	BackendDB *be,
984 	Entry *e,
985 	struct berval *text )
986 {
987 	int rc;
988 	struct mdb_info *mdb;
989 	Operation op = {0};
990 	Opheader ohdr = {0};
991 
992 	assert( be != NULL );
993 	assert( slapMode & SLAP_TOOL_MODE );
994 
995 	assert( text != NULL );
996 	assert( text->bv_val != NULL );
997 	assert( text->bv_val[0] == '\0' );	/* overconservative? */
998 
999 	assert ( e->e_id != NOID );
1000 
1001 	Debug( LDAP_DEBUG_TRACE,
1002 		"=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
1003 		(long) e->e_id, e->e_dn );
1004 
1005 	mdb = (struct mdb_info *) be->be_private;
1006 
1007 	if( cursor ) {
1008 		mdb_cursor_close( cursor );
1009 		cursor = NULL;
1010 	}
1011 	if ( !mdb_tool_txn ) {
1012 		rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
1013 		if( rc != 0 ) {
1014 			snprintf( text->bv_val, text->bv_len,
1015 				"txn_begin failed: %s (%d)",
1016 				mdb_strerror(rc), rc );
1017 			Debug( LDAP_DEBUG_ANY,
1018 				"=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1019 				 text->bv_val );
1020 			return NOID;
1021 		}
1022 	}
1023 
1024 	op.o_hdr = &ohdr;
1025 	op.o_bd = be;
1026 	op.o_tmpmemctx = NULL;
1027 	op.o_tmpmfuncs = &ch_mfuncs;
1028 
1029 	/* id2entry index */
1030 	rc = mdb_id2entry_update( &op, mdb_tool_txn, NULL, e );
1031 	if( rc != 0 ) {
1032 		snprintf( text->bv_val, text->bv_len,
1033 				"id2entry_update failed: err=%d", rc );
1034 		Debug( LDAP_DEBUG_ANY,
1035 			"=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1036 			text->bv_val );
1037 		goto done;
1038 	}
1039 
1040 done:
1041 	if( rc == 0 ) {
1042 		rc = mdb_txn_commit( mdb_tool_txn );
1043 		if( rc != 0 ) {
1044 			mdb->mi_numads = 0;
1045 			snprintf( text->bv_val, text->bv_len,
1046 					"txn_commit failed: %s (%d)",
1047 					mdb_strerror(rc), rc );
1048 			Debug( LDAP_DEBUG_ANY,
1049 				"=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
1050 				"%s\n", text->bv_val );
1051 			e->e_id = NOID;
1052 		}
1053 
1054 	} else {
1055 		mdb_txn_abort( mdb_tool_txn );
1056 		snprintf( text->bv_val, text->bv_len,
1057 			"txn_aborted! %s (%d)",
1058 			mdb_strerror(rc), rc );
1059 		Debug( LDAP_DEBUG_ANY,
1060 			"=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1061 			text->bv_val );
1062 		e->e_id = NOID;
1063 	}
1064 	mdb_tool_txn = NULL;
1065 
1066 	return e->e_id;
1067 }
1068 
mdb_tool_entry_delete(BackendDB * be,struct berval * ndn,struct berval * text)1069 int mdb_tool_entry_delete(
1070 	BackendDB *be,
1071 	struct berval *ndn,
1072 	struct berval *text )
1073 {
1074 	int rc;
1075 	struct mdb_info *mdb;
1076 	Operation op = {0};
1077 	Opheader ohdr = {0};
1078 	Entry *e;
1079 
1080 	assert( be != NULL );
1081 	assert( slapMode & SLAP_TOOL_MODE );
1082 
1083 	assert( text != NULL );
1084 	assert( text->bv_val != NULL );
1085 	assert( text->bv_val[0] == '\0' );	/* overconservative? */
1086 
1087 	assert ( ndn != NULL );
1088 	assert ( ndn->bv_val != NULL );
1089 
1090 	Debug( LDAP_DEBUG_TRACE,
1091 		"=> " LDAP_XSTRING(mdb_tool_entry_delete) "( %s )\n",
1092 		ndn->bv_val );
1093 
1094 	mdb = (struct mdb_info *) be->be_private;
1095 
1096 	assert( cursor == NULL );
1097 	if( cursor ) {
1098 		mdb_cursor_close( cursor );
1099 		cursor = NULL;
1100 	}
1101 	if( !mdb_tool_txn ) {
1102 		rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
1103 		if( rc != 0 ) {
1104 			snprintf( text->bv_val, text->bv_len,
1105 				"txn_begin failed: %s (%d)",
1106 				mdb_strerror(rc), rc );
1107 			Debug( LDAP_DEBUG_ANY,
1108 				"=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1109 				 text->bv_val );
1110 			return LDAP_OTHER;
1111 		}
1112 	}
1113 
1114 	rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &cursor );
1115 	if( rc != 0 ) {
1116 		snprintf( text->bv_val, text->bv_len,
1117 			"cursor_open failed: %s (%d)",
1118 			mdb_strerror(rc), rc );
1119 		Debug( LDAP_DEBUG_ANY,
1120 			"=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1121 			 text->bv_val );
1122 		return LDAP_OTHER;
1123 	}
1124 
1125 	op.o_hdr = &ohdr;
1126 	op.o_bd = be;
1127 	op.o_tmpmemctx = NULL;
1128 	op.o_tmpmfuncs = &ch_mfuncs;
1129 
1130 	rc = mdb_dn2entry( &op, mdb_tool_txn, cursor, ndn, &e, NULL, 0 );
1131 	if( rc != 0 ) {
1132 		snprintf( text->bv_val, text->bv_len,
1133 			"dn2entry failed: %s (%d)",
1134 			mdb_strerror(rc), rc );
1135 		Debug( LDAP_DEBUG_ANY,
1136 			"=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1137 			 text->bv_val );
1138 		goto done;
1139 	}
1140 
1141 	/* check that we wouldn't orphan any children */
1142 	rc = mdb_dn2id_children( &op, mdb_tool_txn, e );
1143 	if( rc != MDB_NOTFOUND ) {
1144 		switch( rc ) {
1145 		case 0:
1146 			snprintf( text->bv_val, text->bv_len,
1147 				"delete failed:"
1148 				" subordinate objects must be deleted first");
1149 			break;
1150 		default:
1151 			snprintf( text->bv_val, text->bv_len,
1152 				"has_children failed: %s (%d)",
1153 				mdb_strerror(rc), rc );
1154 			break;
1155 		}
1156 		rc = -1;
1157 		Debug( LDAP_DEBUG_ANY,
1158 			"=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1159 			 text->bv_val );
1160 		goto done;
1161 	}
1162 
1163 	/* delete from dn2id */
1164 	rc = mdb_dn2id_delete( &op, cursor, e->e_id, 1 );
1165 	if( rc != 0 ) {
1166 		snprintf( text->bv_val, text->bv_len,
1167 				"dn2id_delete failed: err=%d", rc );
1168 		Debug( LDAP_DEBUG_ANY,
1169 			"=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1170 			text->bv_val );
1171 		goto done;
1172 	}
1173 
1174 	/* deindex values */
1175 	rc = mdb_index_entry_del( &op, mdb_tool_txn, e );
1176 	if( rc != 0 ) {
1177 		snprintf( text->bv_val, text->bv_len,
1178 				"entry_delete failed: err=%d", rc );
1179 		Debug( LDAP_DEBUG_ANY,
1180 			"=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1181 			text->bv_val );
1182 		goto done;
1183 	}
1184 
1185 	/* do the deletion */
1186 	rc = mdb_id2entry_delete( be, mdb_tool_txn, e );
1187 	if( rc != 0 ) {
1188 		snprintf( text->bv_val, text->bv_len,
1189 				"id2entry_update failed: err=%d", rc );
1190 		Debug( LDAP_DEBUG_ANY,
1191 			"=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1192 			text->bv_val );
1193 		goto done;
1194 	}
1195 
1196 done:
1197 	/* free entry */
1198 	if( e != NULL ) {
1199 		mdb_entry_return( &op, e );
1200 	}
1201 
1202 	if( rc == 0 ) {
1203 		rc = mdb_txn_commit( mdb_tool_txn );
1204 		if( rc != 0 ) {
1205 			snprintf( text->bv_val, text->bv_len,
1206 					"txn_commit failed: %s (%d)",
1207 					mdb_strerror(rc), rc );
1208 			Debug( LDAP_DEBUG_ANY,
1209 				"=> " LDAP_XSTRING(mdb_tool_entry_delete) ": "
1210 				"%s\n", text->bv_val );
1211 		}
1212 
1213 	} else {
1214 		mdb_txn_abort( mdb_tool_txn );
1215 		snprintf( text->bv_val, text->bv_len,
1216 			"txn_aborted! %s (%d)",
1217 			mdb_strerror(rc), rc );
1218 		Debug( LDAP_DEBUG_ANY,
1219 			"=> " LDAP_XSTRING(mdb_tool_entry_delete) ": %s\n",
1220 			text->bv_val );
1221 	}
1222 	mdb_tool_txn = NULL;
1223 	cursor = NULL;
1224 
1225 	return rc;
1226 }
1227 
1228 static void *
mdb_tool_index_task(void * ctx,void * ptr)1229 mdb_tool_index_task( void *ctx, void *ptr )
1230 {
1231 	int base = *(int *)ptr;
1232 	Operation op = {0};
1233 	Opheader ohdr = {0};
1234 	AttrIxInfo ai = {0}, *aio;
1235 
1236 	free( ptr );
1237 	op.o_hdr = &ohdr;
1238 	op.o_bd = mdb_tool_ix_be;
1239 	op.o_tmpmemctx = NULL;
1240 	op.o_tmpmfuncs = &ch_mfuncs;
1241 	aio = mdb_tool_axinfo[base];
1242 	mdb_tool_axinfo[base] = &ai;
1243 	LDAP_SLIST_INSERT_HEAD( &op.o_extra, &ai.ai_oe, oe_next );
1244 	while ( 1 ) {
1245 		ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
1246 		mdb_tool_index_tcount--;
1247 		if ( !mdb_tool_index_tcount )
1248 			ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1249 		ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
1250 			&mdb_tool_index_mutex );
1251 		if ( slapd_shutdown ) {
1252 			mdb_tool_index_tcount--;
1253 			if ( !mdb_tool_index_tcount )
1254 				ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1255 			*aio = ai;
1256 			mdb_tool_axinfo[base] = aio;
1257 			ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1258 			break;
1259 		}
1260 		ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1261 		mdb_tool_index_rec[base].ir_i = mdb_index_recrun( &op,
1262 			mdb_tool_ix_txn,
1263 			mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
1264 	}
1265 
1266 	return NULL;
1267 }
1268 
1269 #ifdef MDB_TOOL_IDL_CACHING
1270 static int
mdb_tool_idl_cmp(const void * v1,const void * v2)1271 mdb_tool_idl_cmp( const void *v1, const void *v2 )
1272 {
1273 	const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
1274 	int rc;
1275 
1276 	if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
1277 	return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
1278 }
1279 
1280 static int
mdb_tool_idl_flush_one(MDB_cursor * mc,AttrIxInfo * ai,mdb_tool_idl_cache * ic)1281 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrIxInfo *ai, mdb_tool_idl_cache *ic )
1282 {
1283 	mdb_tool_idl_cache_entry *ice;
1284 	MDB_val key, data[2];
1285 	int i, rc;
1286 	ID id, nid;
1287 
1288 	/* Freshly allocated, ignore it */
1289 	if ( !ic->head && ic->count <= MDB_idl_db_size ) {
1290 		return 0;
1291 	}
1292 
1293 	key.mv_data = ic->kstr.bv_val;
1294 	key.mv_size = ic->kstr.bv_len;
1295 
1296 	if ( ic->count > MDB_idl_db_size ) {
1297 		while ( ic->flags & WAS_FOUND ) {
1298 			rc = mdb_cursor_get( mc, &key, data, MDB_SET );
1299 			if ( rc ) {
1300 				/* FIXME: find out why this happens */
1301 				ic->flags = 0;
1302 				break;
1303 			}
1304 			if ( ic->flags & WAS_RANGE ) {
1305 				/* Skip lo */
1306 				rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1307 
1308 				/* Get hi */
1309 				rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1310 
1311 				/* Store range hi */
1312 				data[0].mv_data = &ic->last;
1313 				rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
1314 			} else {
1315 				/* Delete old data, replace with range */
1316 				ic->first = *(ID *)data[0].mv_data;
1317 				mdb_cursor_del( mc, MDB_NODUPDATA );
1318 			}
1319 			break;
1320 		}
1321 		if ( !(ic->flags & WAS_RANGE)) {
1322 			/* range, didn't exist before */
1323 			nid = 0;
1324 			data[0].mv_size = sizeof(ID);
1325 			data[0].mv_data = &nid;
1326 			rc = mdb_cursor_put( mc, &key, data, 0 );
1327 			if ( rc == 0 ) {
1328 				data[0].mv_data = &ic->first;
1329 				rc = mdb_cursor_put( mc, &key, data, 0 );
1330 				if ( rc == 0 ) {
1331 					data[0].mv_data = &ic->last;
1332 					rc = mdb_cursor_put( mc, &key, data, 0 );
1333 				}
1334 			}
1335 			if ( rc ) {
1336 				rc = -1;
1337 			}
1338 		}
1339 	} else {
1340 		/* Normal write */
1341 		int n;
1342 
1343 		data[0].mv_size = sizeof(ID);
1344 		rc = 0;
1345 		for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1346 			int end;
1347 			if ( ice->next ) {
1348 				end = IDBLOCK;
1349 			} else {
1350 				end = (ic->count-ic->offset) & (IDBLOCK-1);
1351 				if ( !end )
1352 					end = IDBLOCK;
1353 			}
1354 			data[1].mv_size = end;
1355 			data[0].mv_data = ice->ids;
1356 			rc = mdb_cursor_put( mc, &key, data, MDB_APPENDDUP|MDB_MULTIPLE );
1357 			if ( rc ) {
1358 				rc = -1;
1359 				break;
1360 			}
1361 		}
1362 		if ( ic->head ) {
1363 			ic->tail->next = ai->ai_flist;
1364 			ai->ai_flist = ic->head;
1365 		}
1366 	}
1367 	ic->head = ai->ai_clist;
1368 	ai->ai_clist = ic;
1369 	return rc;
1370 }
1371 
1372 static int
mdb_tool_idl_flush_db(MDB_txn * txn,AttrInfo * ai,AttrIxInfo * ax)1373 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai, AttrIxInfo *ax )
1374 {
1375 	MDB_cursor *mc;
1376 	Avlnode *root;
1377 	int rc;
1378 
1379 	mdb_cursor_open( txn, ai->ai_dbi, &mc );
1380 	root = ldap_tavl_end( ai->ai_root, TAVL_DIR_LEFT );
1381 	do {
1382 		rc = mdb_tool_idl_flush_one( mc, ax, root->avl_data );
1383 		if ( rc != -1 )
1384 			rc = 0;
1385 	} while ((root = ldap_tavl_next(root, TAVL_DIR_RIGHT)));
1386 	mdb_cursor_close( mc );
1387 
1388 	return rc;
1389 }
1390 
1391 static int
mdb_tool_idl_flush(BackendDB * be,MDB_txn * txn)1392 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
1393 {
1394 	struct mdb_info *mdb = (struct mdb_info *) be->be_private;
1395 	int rc = 0;
1396 	unsigned int i, dbi;
1397 
1398 	for ( i=0; i < mdb->mi_nattrs; i++ ) {
1399 		if ( !mdb->mi_attrs[i]->ai_root ) continue;
1400 		rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i], mdb_tool_axinfo[i % mdb_tool_threads] );
1401 		ldap_tavl_free(mdb->mi_attrs[i]->ai_root, NULL);
1402 		mdb->mi_attrs[i]->ai_root = NULL;
1403 		if ( rc )
1404 			break;
1405 	}
1406 	return rc;
1407 }
1408 
mdb_tool_idl_add(BackendDB * be,MDB_cursor * mc,struct berval * keys,ID id)1409 int mdb_tool_idl_add(
1410 	BackendDB *be,
1411 	MDB_cursor *mc,
1412 	struct berval *keys,
1413 	ID id )
1414 {
1415 	MDB_dbi dbi;
1416 	mdb_tool_idl_cache *ic, itmp;
1417 	mdb_tool_idl_cache_entry *ice;
1418 	int i, rc, lcount;
1419 	AttrIxInfo *ax = (AttrIxInfo *)mc;
1420 	AttrInfo *ai = (AttrInfo *)ax->ai_ai;
1421 	mc = ai->ai_cursor;
1422 
1423 	dbi = ai->ai_dbi;
1424 	for (i=0; keys[i].bv_val; i++) {
1425 	itmp.kstr = keys[i];
1426 	ic = ldap_tavl_find( ai->ai_root, &itmp, mdb_tool_idl_cmp );
1427 
1428 	/* No entry yet, create one */
1429 	if ( !ic ) {
1430 		MDB_val key, data;
1431 		ID nid;
1432 		int rc;
1433 
1434 		if ( ax->ai_clist ) {
1435 			ic = ax->ai_clist;
1436 			ax->ai_clist = ic->head;
1437 		} else {
1438 			ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
1439 		}
1440 		ic->kstr.bv_len = itmp.kstr.bv_len;
1441 		ic->kstr.bv_val = (char *)(ic+1);
1442 		memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1443 		ic->head = ic->tail = NULL;
1444 		ic->last = 0;
1445 		ic->count = 0;
1446 		ic->offset = 0;
1447 		ic->flags = 0;
1448 		ldap_tavl_insert( &ai->ai_root, ic, mdb_tool_idl_cmp,
1449 			ldap_avl_dup_error );
1450 
1451 		/* load existing key count here */
1452 		key.mv_size = keys[i].bv_len;
1453 		key.mv_data = keys[i].bv_val;
1454 		rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
1455 		if ( rc == 0 ) {
1456 			ic->flags |= WAS_FOUND;
1457 			nid = *(ID *)data.mv_data;
1458 			if ( nid == 0 ) {
1459 				ic->count = MDB_idl_db_size+1;
1460 				ic->flags |= WAS_RANGE;
1461 			} else {
1462 				size_t count;
1463 
1464 				mdb_cursor_count( mc, &count );
1465 				ic->count = count;
1466 				ic->first = nid;
1467 				ic->offset = count & (IDBLOCK-1);
1468 			}
1469 		}
1470 	}
1471 	/* are we a range already? */
1472 	if ( ic->count > MDB_idl_db_size ) {
1473 		ic->last = id;
1474 		continue;
1475 	/* Are we at the limit, and converting to a range? */
1476 	} else if ( ic->count == MDB_idl_db_size ) {
1477 		if ( ic->head ) {
1478 			ic->tail->next = ax->ai_flist;
1479 			ax->ai_flist = ic->head;
1480 		}
1481 		ic->head = ic->tail = NULL;
1482 		ic->last = id;
1483 		ic->count++;
1484 		continue;
1485 	}
1486 	/* No free block, create that too */
1487 	lcount = (ic->count-ic->offset) & (IDBLOCK-1);
1488 	if ( !ic->tail || lcount == 0) {
1489 		if ( ax->ai_flist ) {
1490 			ice = ax->ai_flist;
1491 			ax->ai_flist = ice->next;
1492 		} else {
1493 			ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
1494 		}
1495 		ice->next = NULL;
1496 		if ( !ic->head ) {
1497 			ic->head = ice;
1498 		} else {
1499 			ic->tail->next = ice;
1500 		}
1501 		ic->tail = ice;
1502 		if ( lcount )
1503 			ice->ids[lcount-1] = 0;
1504 		if ( !ic->count )
1505 			ic->first = id;
1506 	}
1507 	ice = ic->tail;
1508 	if (!lcount || ice->ids[lcount-1] != id) {
1509 		ice->ids[lcount] = id;
1510 		ic->count++;
1511 	}
1512 	}
1513 
1514 	return 0;
1515 }
1516 #endif /* MDB_TOOL_IDL_CACHING */
1517 
1518 /* Upgrade from pre 2.4.34 dn2id format */
1519 
1520 #include <ac/unistd.h>
1521 #include <lutil_meter.h>
1522 
1523 #define STACKSIZ	2048
1524 
1525 typedef struct rec {
1526 	ID id;
1527 	size_t len;
1528 	char rdn[512];
1529 } rec;
1530 
1531 static int
mdb_dn2id_upgrade(BackendDB * be)1532 mdb_dn2id_upgrade( BackendDB *be ) {
1533 	struct mdb_info *mi = (struct mdb_info *) be->be_private;
1534 	MDB_txn *mt;
1535 	MDB_cursor *mc = NULL;
1536 	MDB_val key, data;
1537 	int rc, writes=0, depth=0;
1538 	int enable_meter = 0;
1539 	ID id = 0, *num, count = 0;
1540 	rec *stack;
1541 	lutil_meter_t meter;
1542 
1543 	if (!(mi->mi_flags & MDB_NEED_UPGRADE)) {
1544 		Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n",
1545 			be->be_suffix[0].bv_val );
1546 		return 0;
1547 	}
1548 
1549 	{
1550 		MDB_stat st;
1551 
1552 		mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st);
1553 		if (!st.ms_entries) {
1554 			/* Empty DB, nothing to upgrade? */
1555 			return 0;
1556 		}
1557 		if (isatty(2))
1558 			enable_meter = !lutil_meter_open(&meter,
1559 				&lutil_meter_text_display,
1560 				&lutil_meter_linear_estimator,
1561 				st.ms_entries);
1562 	}
1563 
1564 	num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec)));
1565 	stack = (rec *)(num + STACKSIZ);
1566 
1567 	rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1568 	if (rc) {
1569 		Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n",
1570 			mdb_strerror(rc), rc );
1571 		goto leave;
1572 	}
1573 	rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1574 	if (rc) {
1575 		Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n",
1576 			mdb_strerror(rc), rc );
1577 		goto leave;
1578 	}
1579 
1580 	key.mv_size = sizeof(ID);
1581 	/* post-order depth-first update */
1582 	for(;;) {
1583 		size_t dkids;
1584 		unsigned char *ptr;
1585 
1586 		/* visit */
1587 		key.mv_data = &id;
1588 		stack[depth].id = id;
1589 		rc = mdb_cursor_get(mc, &key, &data, MDB_SET);
1590 		if (rc) {
1591 			Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n",
1592 				mdb_strerror(rc), rc );
1593 			goto leave;
1594 		}
1595 		num[depth] = 1;
1596 
1597 		rc = mdb_cursor_count(mc, &dkids);
1598 		if (rc) {
1599 			Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n",
1600 				mdb_strerror(rc), rc );
1601 			goto leave;
1602 		}
1603 		if (dkids > 1) {
1604 			rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1605 down:
1606 			ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID);
1607 			memcpy(&id, ptr, sizeof(ID));
1608 			depth++;
1609 			memcpy(stack[depth].rdn, data.mv_data, data.mv_size);
1610 			stack[depth].len = data.mv_size;
1611 			continue;
1612 		}
1613 
1614 
1615 		/* pop: write updated count, advance to next node */
1616 pop:
1617 		/* update superior counts */
1618 		if (depth)
1619 			num[depth-1] += num[depth];
1620 
1621 		key.mv_data = &id;
1622 		id = stack[depth-1].id;
1623 		data.mv_data = stack[depth].rdn;
1624 		data.mv_size = stack[depth].len;
1625 		rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1626 		if (rc) {
1627 			Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n",
1628 				mdb_strerror(rc), rc );
1629 			goto leave;
1630 		}
1631 		data.mv_data = stack[depth].rdn;
1632 		ptr = (unsigned char *)data.mv_data + data.mv_size;
1633 		memcpy(ptr, &num[depth], sizeof(ID));
1634 		data.mv_size += sizeof(ID);
1635 		rc = mdb_cursor_del(mc, 0);
1636 		if (rc) {
1637 			Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n",
1638 				mdb_strerror(rc), rc );
1639 			goto leave;
1640 		}
1641 		rc = mdb_cursor_put(mc, &key, &data, 0);
1642 		if (rc) {
1643 			Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n",
1644 				mdb_strerror(rc), rc );
1645 			goto leave;
1646 		}
1647 		count++;
1648 #if 1
1649 		if (enable_meter)
1650 			lutil_meter_update(&meter, count, 0);
1651 #else
1652 		{
1653 			int len;
1654 			ptr = data.mv_data;
1655 			len = (ptr[0] & 0x7f) << 8 | ptr[1];
1656 			printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2);
1657 		}
1658 #endif
1659 		writes++;
1660 		if (writes == 1000) {
1661 			mdb_cursor_close(mc);
1662 			rc = mdb_txn_commit(mt);
1663 			if (rc) {
1664 				Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n",
1665 					mdb_strerror(rc), rc );
1666 				goto leave;
1667 			}
1668 			rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1669 			if (rc) {
1670 				Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n",
1671 					mdb_strerror(rc), rc );
1672 				goto leave;
1673 			}
1674 			rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1675 			if (rc) {
1676 				Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n",
1677 					mdb_strerror(rc), rc );
1678 				goto leave;
1679 			}
1680 			rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1681 			if (rc) {
1682 				Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n",
1683 					mdb_strerror(rc), rc );
1684 				goto leave;
1685 			}
1686 			writes = 0;
1687 		}
1688 		depth--;
1689 
1690 		rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1691 		if (rc == 0)
1692 			goto down;
1693 		rc = 0;
1694 		if (depth)
1695 			goto pop;
1696 		else
1697 			break;
1698 	}
1699 leave:
1700 	mdb_cursor_close(mc);
1701 	if (mt) {
1702 		int r2;
1703 		r2 = mdb_txn_commit(mt);
1704 		if (r2) {
1705 			Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n",
1706 				mdb_strerror(r2), r2 );
1707 			if (!rc)
1708 				rc = r2;
1709 		}
1710 	}
1711 	ch_free(num);
1712 	if (enable_meter) {
1713 		lutil_meter_update(&meter, count, 1);
1714 		lutil_meter_close(&meter);
1715 	}
1716 	return rc;
1717 }
1718