1 /* $NetBSD: apropos-utils.c,v 1.48 2021/11/27 22:30:25 rillig Exp $ */ 2 /*- 3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com> 4 * All rights reserved. 5 * 6 * This code was developed as part of Google's Summer of Code 2011 program. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __RCSID("$NetBSD: apropos-utils.c,v 1.48 2021/11/27 22:30:25 rillig Exp $"); 35 36 #include <sys/queue.h> 37 #include <sys/stat.h> 38 39 #include <assert.h> 40 #include <ctype.h> 41 #include <err.h> 42 #include <math.h> 43 #include <stdio.h> 44 #include <stdlib.h> 45 #include <string.h> 46 #include <util.h> 47 #include <zlib.h> 48 #include <term.h> 49 #include <unistd.h> 50 #undef tab // XXX: manconf.h 51 52 #include "apropos-utils.h" 53 #include "custom_apropos_tokenizer.h" 54 #include "manconf.h" 55 #include "fts3_tokenizer.h" 56 57 typedef struct orig_callback_data { 58 void *data; 59 int (*callback) (query_callback_args*); 60 } orig_callback_data; 61 62 typedef struct inverse_document_frequency { 63 double value; 64 int status; 65 } inverse_document_frequency; 66 67 /* weights for individual columns */ 68 static const double col_weights[] = { 69 2.0, // NAME 70 2.00, // Name-description 71 0.55, // DESCRIPTION 72 0.10, // LIBRARY 73 0.001, //RETURN VALUES 74 0.20, //ENVIRONMENT 75 0.01, //FILES 76 0.001, //EXIT STATUS 77 2.00, //DIAGNOSTICS 78 0.05, //ERRORS 79 0.00, //md5_hash 80 1.00 //machine 81 }; 82 83 #ifndef APROPOS_DEBUG 84 static int 85 register_tokenizer(sqlite3 *db) 86 { 87 int rc; 88 sqlite3_stmt *stmt; 89 const sqlite3_tokenizer_module *p; 90 const char *name = "custom_apropos_tokenizer"; 91 get_custom_apropos_tokenizer(&p); 92 const char *sql = "SELECT fts3_tokenizer(?, ?)"; 93 94 sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); 95 rc = sqlite3_prepare_v2(db, sql, -1, &stmt, 0); 96 if (rc != SQLITE_OK) 97 return rc; 98 99 sqlite3_bind_text(stmt, 1, name, -1, SQLITE_STATIC); 100 sqlite3_bind_blob(stmt, 2, &p, sizeof(p), SQLITE_STATIC); 101 sqlite3_step(stmt); 102 103 return sqlite3_finalize(stmt); 104 } 105 #endif 106 107 /* 108 * lower -- 109 * Converts the string str to lower case 110 */ 111 char * 112 lower(char *str) 113 { 114 assert(str); 115 int i = 0; 116 char c; 117 while ((c = str[i]) != '\0') 118 str[i++] = tolower((unsigned char) c); 119 return str; 120 } 121 122 /* 123 * concat-- 124 * Utility function. Concatenates together: dst, a space character and src. 125 * dst + " " + src 126 */ 127 void 128 concat(char **dst, const char *src) 129 { 130 concat2(dst, src, strlen(src)); 131 } 132 133 void 134 concat2(char **dst, const char *src, size_t srclen) 135 { 136 size_t totallen, dstlen; 137 char *mydst = *dst; 138 assert(src != NULL); 139 140 /* 141 * If destination buffer dst is NULL, then simply 142 * strdup the source buffer 143 */ 144 if (mydst == NULL) { 145 mydst = estrndup(src, srclen); 146 *dst = mydst; 147 return; 148 } 149 150 dstlen = strlen(mydst); 151 /* 152 * NUL Byte and separator space 153 */ 154 totallen = dstlen + srclen + 2; 155 156 mydst = erealloc(mydst, totallen); 157 158 /* Append a space at the end of dst */ 159 mydst[dstlen++] = ' '; 160 161 /* Now, copy src at the end of dst */ 162 memcpy(mydst + dstlen, src, srclen); 163 mydst[dstlen + srclen] = '\0'; 164 *dst = mydst; 165 } 166 167 void 168 close_db(sqlite3 *db) 169 { 170 sqlite3_close(db); 171 sqlite3_shutdown(); 172 } 173 174 /* 175 * create_db -- 176 * Creates the database schema. 177 */ 178 static int 179 create_db(sqlite3 *db) 180 { 181 const char *sqlstr = NULL; 182 char *schemasql; 183 char *errmsg = NULL; 184 185 /*------------------------ Create the tables------------------------------*/ 186 187 #if NOTYET 188 sqlite3_exec(db, "PRAGMA journal_mode = WAL", NULL, NULL, NULL); 189 #else 190 sqlite3_exec(db, "PRAGMA journal_mode = DELETE", NULL, NULL, NULL); 191 #endif 192 193 schemasql = sqlite3_mprintf("PRAGMA user_version = %d", 194 APROPOS_SCHEMA_VERSION); 195 sqlite3_exec(db, schemasql, NULL, NULL, &errmsg); 196 if (errmsg != NULL) 197 goto out; 198 sqlite3_free(schemasql); 199 200 sqlstr = 201 //mandb 202 "CREATE VIRTUAL TABLE mandb USING fts4(section, name, " 203 "name_desc, desc, lib, return_vals, env, files, " 204 "exit_status, diagnostics, errors, md5_hash UNIQUE, machine, " 205 #ifndef APROPOS_DEBUG 206 "compress=zip, uncompress=unzip, tokenize=custom_apropos_tokenizer, " 207 #else 208 "tokenize=porter, " 209 #endif 210 "notindexed=section, notindexed=md5_hash); " 211 //mandb_meta 212 "CREATE TABLE IF NOT EXISTS mandb_meta(device, inode, mtime, " 213 "file UNIQUE, md5_hash UNIQUE, id INTEGER PRIMARY KEY); " 214 //mandb_links 215 "CREATE TABLE IF NOT EXISTS mandb_links(link COLLATE NOCASE, target, section, " 216 "machine, md5_hash, name_desc); "; 217 218 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 219 if (errmsg != NULL) 220 goto out; 221 222 sqlstr = 223 "CREATE INDEX IF NOT EXISTS index_mandb_links ON mandb_links " 224 "(link); " 225 "CREATE INDEX IF NOT EXISTS index_mandb_meta_dev ON mandb_meta " 226 "(device, inode); " 227 "CREATE INDEX IF NOT EXISTS index_mandb_links_md5 ON mandb_links " 228 "(md5_hash);"; 229 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 230 if (errmsg != NULL) 231 goto out; 232 return 0; 233 234 out: 235 warnx("%s", errmsg); 236 free(errmsg); 237 sqlite3_close(db); 238 sqlite3_shutdown(); 239 return -1; 240 } 241 242 /* 243 * zip -- 244 * User defined Sqlite function to compress the FTS table 245 */ 246 static void 247 zip(sqlite3_context *pctx, int nval, sqlite3_value **apval) 248 { 249 int nin; 250 long int nout; 251 const unsigned char * inbuf; 252 unsigned char *outbuf; 253 254 assert(nval == 1); 255 nin = sqlite3_value_bytes(apval[0]); 256 inbuf = (const unsigned char *) sqlite3_value_blob(apval[0]); 257 nout = nin + 13 + (nin + 999) / 1000; 258 outbuf = emalloc(nout); 259 compress(outbuf, (unsigned long *) &nout, inbuf, nin); 260 sqlite3_result_blob(pctx, outbuf, nout, free); 261 } 262 263 /* 264 * unzip -- 265 * User defined Sqlite function to uncompress the FTS table. 266 */ 267 static void 268 unzip(sqlite3_context *pctx, int nval, sqlite3_value **apval) 269 { 270 unsigned int rc; 271 unsigned char *outbuf; 272 z_stream stream; 273 long total_out; 274 275 assert(nval == 1); 276 memset(&stream, 0, sizeof(stream)); 277 stream.next_in = __UNCONST(sqlite3_value_blob(apval[0])); 278 stream.avail_in = sqlite3_value_bytes(apval[0]); 279 stream.zalloc = NULL; 280 stream.zfree = NULL; 281 282 if (inflateInit(&stream) != Z_OK) { 283 return; 284 } 285 286 total_out = stream.avail_out = stream.avail_in * 2 + 100; 287 stream.next_out = outbuf = emalloc(stream.avail_out); 288 while ((rc = inflate(&stream, Z_SYNC_FLUSH)) != Z_STREAM_END) { 289 if (rc != Z_OK || 290 (stream.avail_out != 0 && stream.avail_in == 0)) { 291 free(outbuf); 292 return; 293 } 294 total_out <<= 1; 295 outbuf = erealloc(outbuf, total_out); 296 stream.next_out = outbuf + stream.total_out; 297 stream.avail_out = total_out - stream.total_out; 298 } 299 if (inflateEnd(&stream) != Z_OK) { 300 free(outbuf); 301 return; 302 } 303 if (stream.total_out == 0) { 304 free(outbuf); 305 return; 306 } 307 outbuf = erealloc(outbuf, stream.total_out); 308 sqlite3_result_text(pctx, (const char *)outbuf, stream.total_out, free); 309 } 310 311 /* 312 * get_dbpath -- 313 * Read the path of the database from man.conf and return. 314 */ 315 char * 316 get_dbpath(const char *manconf) 317 { 318 TAG *tp; 319 char *dbpath; 320 321 config(manconf); 322 tp = gettag("_mandb", 1); 323 if (!tp) 324 return NULL; 325 326 if (TAILQ_EMPTY(&tp->entrylist)) 327 return NULL; 328 329 dbpath = TAILQ_LAST(&tp->entrylist, tqh)->s; 330 return dbpath; 331 } 332 333 /* init_db -- 334 * Prepare the database. Register the compress/uncompress functions and the 335 * stopword tokenizer. 336 * db_flag specifies the mode in which to open the database. 3 options are 337 * available: 338 * 1. DB_READONLY: Open in READONLY mode. An error if db does not exist. 339 * 2. DB_READWRITE: Open in read-write mode. An error if db does not exist. 340 * 3. DB_CREATE: Open in read-write mode. It will try to create the db if 341 * it does not exist already. 342 * RETURN VALUES: 343 * The function will return NULL in case the db does not exist 344 * and DB_CREATE 345 * was not specified. And in case DB_CREATE was specified and yet NULL is 346 * returned, then there was some other error. 347 * In normal cases the function should return a handle to the db. 348 */ 349 sqlite3 * 350 init_db(mandb_access_mode db_flag, const char *manconf) 351 { 352 sqlite3 *db = NULL; 353 sqlite3_stmt *stmt; 354 struct stat sb; 355 int rc; 356 int create_db_flag = 0; 357 358 char *dbpath = get_dbpath(manconf); 359 if (dbpath == NULL) 360 errx(EXIT_FAILURE, "_mandb entry not found in man.conf"); 361 362 if (!(stat(dbpath, &sb) == 0 && S_ISREG(sb.st_mode))) { 363 /* Database does not exist, check if DB_CREATE was specified, 364 * and set flag to create the database schema 365 */ 366 if (db_flag != (MANDB_CREATE)) { 367 warnx("Missing apropos database. " 368 "Please run makemandb to create it."); 369 return NULL; 370 } 371 create_db_flag = 1; 372 } else { 373 /* 374 * Database exists. Check if we have the permissions 375 * to read/write the files 376 */ 377 int access_mode = R_OK; 378 switch (db_flag) { 379 case MANDB_CREATE: 380 case MANDB_WRITE: 381 access_mode |= W_OK; 382 break; 383 default: 384 break; 385 } 386 if ((access(dbpath, access_mode)) != 0) { 387 warnx("Unable to access the database, please check" 388 " permissions for `%s'", dbpath); 389 return NULL; 390 } 391 } 392 393 sqlite3_initialize(); 394 rc = sqlite3_open_v2(dbpath, &db, db_flag, NULL); 395 396 if (rc != SQLITE_OK) { 397 warnx("%s", sqlite3_errmsg(db)); 398 goto error; 399 } 400 401 sqlite3_extended_result_codes(db, 1); 402 403 #ifndef APROPOS_DEBUG 404 rc = register_tokenizer(db); 405 if (rc != SQLITE_OK) { 406 warnx("Unable to register custom tokenizer: %s", sqlite3_errmsg(db)); 407 goto error; 408 } 409 #endif 410 411 if (create_db_flag && create_db(db) < 0) { 412 warnx("%s", "Unable to create database schema"); 413 goto error; 414 } 415 416 rc = sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &stmt, NULL); 417 if (rc != SQLITE_OK) { 418 warnx("Unable to query schema version: %s", 419 sqlite3_errmsg(db)); 420 goto error; 421 } 422 if (sqlite3_step(stmt) != SQLITE_ROW) { 423 sqlite3_finalize(stmt); 424 warnx("Unable to query schema version: %s", 425 sqlite3_errmsg(db)); 426 goto error; 427 } 428 if (sqlite3_column_int(stmt, 0) != APROPOS_SCHEMA_VERSION) { 429 sqlite3_finalize(stmt); 430 warnx("Incorrect schema version found. " 431 "Please run makemandb -f."); 432 goto error; 433 } 434 sqlite3_finalize(stmt); 435 436 437 /* Register the zip and unzip functions for FTS compression */ 438 rc = sqlite3_create_function(db, "zip", 1, SQLITE_ANY, NULL, zip, 439 NULL, NULL); 440 if (rc != SQLITE_OK) { 441 warnx("Unable to register function: compress: %s", 442 sqlite3_errmsg(db)); 443 goto error; 444 } 445 446 rc = sqlite3_create_function(db, "unzip", 1, SQLITE_ANY, NULL, 447 unzip, NULL, NULL); 448 if (rc != SQLITE_OK) { 449 warnx("Unable to register function: uncompress: %s", 450 sqlite3_errmsg(db)); 451 goto error; 452 } 453 return db; 454 455 error: 456 close_db(db); 457 return NULL; 458 } 459 460 /* 461 * rank_func -- 462 * Sqlite user defined function for ranking the documents. 463 * For each phrase of the query, it computes the tf and idf and adds them over. 464 * It computes the final rank, by multiplying tf and idf together. 465 * Weight of term t for document d = (term frequency of t in d * 466 * inverse document frequency of t) 467 * 468 * Term Frequency of term t in document d = Number of times t occurs in d / 469 * Number of times t appears in all documents 470 * 471 * Inverse document frequency of t = log(Total number of documents / 472 * Number of documents in which t occurs) 473 */ 474 static void 475 rank_func(sqlite3_context *pctx, int nval, sqlite3_value **apval) 476 { 477 inverse_document_frequency *idf = sqlite3_user_data(pctx); 478 double tf = 0.0; 479 const unsigned int *matchinfo; 480 int ncol; 481 int nphrase; 482 int iphrase; 483 int ndoc; 484 int doclen = 0; 485 const double k = 3.75; 486 /* 487 * Check that the number of arguments passed to this 488 * function is correct. 489 */ 490 assert(nval == 1); 491 492 matchinfo = (const unsigned int *) sqlite3_value_blob(apval[0]); 493 nphrase = matchinfo[0]; 494 ncol = matchinfo[1]; 495 ndoc = matchinfo[2 + 3 * ncol * nphrase + ncol]; 496 for (iphrase = 0; iphrase < nphrase; iphrase++) { 497 int icol; 498 const unsigned int *phraseinfo = 499 &matchinfo[2 + ncol + iphrase * ncol * 3]; 500 for(icol = 1; icol < ncol; icol++) { 501 502 /* nhitcount: number of times the current phrase occurs 503 * in the current column in the current document. 504 * nglobalhitcount: number of times current phrase 505 * occurs in the current column in all documents. 506 * ndocshitcount: number of documents in which the 507 * current phrase occurs in the current column at 508 * least once. 509 */ 510 int nhitcount = phraseinfo[3 * icol]; 511 int nglobalhitcount = phraseinfo[3 * icol + 1]; 512 int ndocshitcount = phraseinfo[3 * icol + 2]; 513 doclen = matchinfo[2 + icol ]; 514 double weight = col_weights[icol - 1]; 515 if (idf->status == 0 && ndocshitcount) 516 idf->value += 517 log(((double)ndoc / ndocshitcount))* weight; 518 519 /* 520 * Dividing the tf by document length to normalize 521 * the effect of longer documents. 522 */ 523 if (nglobalhitcount > 0 && nhitcount) 524 tf += (((double)nhitcount * weight) 525 / (nglobalhitcount * doclen)); 526 } 527 } 528 idf->status = 1; 529 530 /* 531 * Final score: Dividing by k + tf further normalizes the weight 532 * leading to better results. The value of k is experimental 533 */ 534 double score = (tf * idf->value) / (k + tf); 535 sqlite3_result_double(pctx, score); 536 return; 537 } 538 539 /* 540 * generates sql query for matching the user entered query 541 */ 542 static char * 543 generate_search_query(query_args *args, const char *snippet_args[3]) 544 { 545 const char *default_snippet_args[3]; 546 char *section_clause = NULL; 547 char *limit_clause = NULL; 548 char *machine_clause = NULL; 549 char *query = NULL; 550 551 if (args->machine) { 552 machine_clause = sqlite3_mprintf("AND mandb.machine=%Q", args->machine); 553 if (machine_clause == NULL) 554 goto RETURN; 555 } 556 557 if (args->nrec >= 0) { 558 /* Use the provided number of records and offset */ 559 limit_clause = sqlite3_mprintf(" LIMIT %d OFFSET %d", 560 args->nrec, args->offset); 561 if (limit_clause == NULL) 562 goto RETURN; 563 } 564 565 /* We want to build a query of the form: "select x,y,z from mandb where 566 * mandb match :query [AND (section IN ('1', '2')] 567 * ORDER BY rank DESC [LIMIT 10 OFFSET 0]" 568 * NOTES: 569 * 1. The portion in first pair of square brackets is optional. 570 * It will be there only if the user has specified an option 571 * to search in one or more specific sections. 572 * 2. The LIMIT portion will be there if the user has specified 573 * a limit using the -n option. 574 */ 575 if (args->sections && args->sections[0]) { 576 concat(§ion_clause, " AND mandb.section IN ("); 577 for (size_t i = 0; args->sections[i]; i++) { 578 char *temp; 579 char c = args->sections[i + 1]? ',': ')'; 580 if ((temp = sqlite3_mprintf("%Q%c", args->sections[i], c)) == NULL) 581 goto RETURN; 582 concat(§ion_clause, temp); 583 sqlite3_free(temp); 584 } 585 } 586 587 if (snippet_args == NULL) { 588 default_snippet_args[0] = ""; 589 default_snippet_args[1] = ""; 590 default_snippet_args[2] = "..."; 591 snippet_args = default_snippet_args; 592 } 593 594 if (args->legacy) { 595 char *wild; 596 easprintf(&wild, "%%%s%%", args->search_str); 597 query = sqlite3_mprintf("SELECT section, name, name_desc, machine" 598 " FROM mandb" 599 " WHERE name LIKE %Q OR name_desc LIKE %Q " 600 "%s" 601 "%s", 602 wild, wild, 603 section_clause ? section_clause : "", 604 limit_clause ? limit_clause : ""); 605 free(wild); 606 } else if (strchr(args->search_str, ' ') == NULL) { 607 /* 608 * If it's a single word query, we want to search in the 609 * links table as well. If the link table contains an entry 610 * for the queried keyword, we want to use that as the name of 611 * the man page. 612 * For example, for `apropos realloc` the output should be 613 * realloc(3) and not malloc(3). 614 */ 615 query = sqlite3_mprintf( 616 "SELECT section, name, name_desc, machine," 617 " snippet(mandb, %Q, %Q, %Q, -1, 40 )," 618 " rank_func(matchinfo(mandb, \"pclxn\")) AS rank" 619 " FROM mandb WHERE name NOT IN (" 620 " SELECT target FROM mandb_links WHERE link=%Q AND" 621 " mandb_links.section=mandb.section) AND mandb MATCH %Q %s %s" 622 " UNION" 623 " SELECT mandb.section, mandb_links.link AS name, mandb.name_desc," 624 " mandb.machine, '' AS snippet, 100.00 AS rank" 625 " FROM mandb JOIN mandb_links ON mandb.name=mandb_links.target and" 626 " mandb.section=mandb_links.section WHERE mandb_links.link=%Q" 627 " %s %s" 628 " ORDER BY rank DESC %s", 629 snippet_args[0], snippet_args[1], snippet_args[2], 630 args->search_str, args->search_str, section_clause ? section_clause : "", 631 machine_clause ? machine_clause : "", args->search_str, 632 machine_clause ? machine_clause : "", 633 section_clause ? section_clause : "", 634 limit_clause ? limit_clause : ""); 635 } else { 636 query = sqlite3_mprintf("SELECT section, name, name_desc, machine," 637 " snippet(mandb, %Q, %Q, %Q, -1, 40 )," 638 " rank_func(matchinfo(mandb, \"pclxn\")) AS rank" 639 " FROM mandb" 640 " WHERE mandb MATCH %Q %s " 641 "%s" 642 " ORDER BY rank DESC" 643 "%s", 644 snippet_args[0], snippet_args[1], snippet_args[2], 645 args->search_str, machine_clause ? machine_clause : "", 646 section_clause ? section_clause : "", 647 limit_clause ? limit_clause : ""); 648 } 649 650 RETURN: 651 sqlite3_free(machine_clause); 652 free(section_clause); 653 sqlite3_free(limit_clause); 654 return query; 655 } 656 657 static const char * 658 get_stmt_col_text(sqlite3_stmt *stmt, int col) 659 { 660 const char *t = (const char *) sqlite3_column_text(stmt, col); 661 return t == NULL ? "*?*" : t; 662 } 663 664 /* 665 * Execute the full text search query and return the number of results 666 * obtained. 667 */ 668 static unsigned int 669 execute_search_query(sqlite3 *db, char *query, query_args *args) 670 { 671 sqlite3_stmt *stmt; 672 char *name; 673 char *slash_ptr; 674 const char *name_temp; 675 char *m = NULL; 676 int rc; 677 query_callback_args callback_args; 678 inverse_document_frequency idf = {0, 0}; 679 680 if (!args->legacy) { 681 /* Register the rank function */ 682 rc = sqlite3_create_function(db, "rank_func", 1, SQLITE_ANY, 683 (void *) &idf, rank_func, NULL, NULL); 684 if (rc != SQLITE_OK) { 685 warnx("Unable to register the ranking function: %s", 686 sqlite3_errmsg(db)); 687 sqlite3_close(db); 688 sqlite3_shutdown(); 689 exit(EXIT_FAILURE); 690 } 691 } 692 693 rc = sqlite3_prepare_v2(db, query, -1, &stmt, NULL); 694 if (rc == SQLITE_IOERR) { 695 warnx("Corrupt database. Please rerun makemandb"); 696 return -1; 697 } else if (rc != SQLITE_OK) { 698 warnx("%s", sqlite3_errmsg(db)); 699 return -1; 700 } 701 702 unsigned int nresults = 0; 703 while (sqlite3_step(stmt) == SQLITE_ROW) { 704 nresults++; 705 callback_args.section = get_stmt_col_text(stmt, 0); 706 name_temp = get_stmt_col_text(stmt, 1); 707 callback_args.name_desc = get_stmt_col_text(stmt, 2); 708 callback_args.machine = (const char *) sqlite3_column_text(stmt, 3); 709 if (!args->legacy) { 710 callback_args.snippet = get_stmt_col_text(stmt, 4); 711 callback_args.snippet_length = 712 strlen(callback_args.snippet); 713 } else { 714 callback_args.snippet = ""; 715 callback_args.snippet_length = 1; 716 } 717 if ((slash_ptr = strrchr(name_temp, '/')) != NULL) 718 name_temp = slash_ptr + 1; 719 if (callback_args.machine && callback_args.machine[0]) { 720 m = estrdup(callback_args.machine); 721 easprintf(&name, "%s/%s", lower(m), name_temp); 722 free(m); 723 } else { 724 name = estrdup(get_stmt_col_text(stmt, 1)); 725 } 726 callback_args.name = name; 727 callback_args.other_data = args->callback_data; 728 (args->callback)(&callback_args); 729 free(name); 730 } 731 sqlite3_finalize(stmt); 732 return nresults; 733 } 734 735 736 /* 737 * run_query_internal -- 738 * Performs the searches for the keywords entered by the user. 739 * The 2nd param: snippet_args is an array of strings providing values for the 740 * last three parameters to the snippet function of sqlite. (Look at the docs). 741 * The 3rd param: args contains rest of the search parameters. Look at 742 * arpopos-utils.h for the description of individual fields. 743 * 744 */ 745 static int 746 run_query_internal(sqlite3 *db, const char *snippet_args[3], query_args *args) 747 { 748 char *query; 749 query = generate_search_query(args, snippet_args); 750 if (query == NULL) { 751 *args->errmsg = estrdup("malloc failed"); 752 return -1; 753 } 754 755 execute_search_query(db, query, args); 756 sqlite3_free(query); 757 return *(args->errmsg) == NULL ? 0 : -1; 758 } 759 760 static char * 761 get_escaped_html_string(const char *src, size_t *slen) 762 { 763 static const char trouble[] = "<>\"&\002\003"; 764 /* 765 * First scan the src to find out the number of occurrences 766 * of {'>', '<' '"', '&'}. Then allocate a new buffer with 767 * sufficient space to be able to store the quoted versions 768 * of the special characters {>, <, ", &}. 769 * Copy over the characters from the original src into 770 * this buffer while replacing the special characters with 771 * their quoted versions. 772 */ 773 char *dst, *ddst; 774 size_t count; 775 const char *ssrc; 776 777 for (count = 0, ssrc = src; *src; count++) { 778 size_t sz = strcspn(src, trouble); 779 src += sz + 1; 780 } 781 782 783 #define append(a) \ 784 do { \ 785 memcpy(dst, (a), sizeof(a) - 1); \ 786 dst += sizeof(a) - 1; \ 787 } while (0) 788 789 790 ddst = dst = emalloc(*slen + count * 5 + 1); 791 for (src = ssrc; *src; src++) { 792 switch (*src) { 793 case '<': 794 append("<"); 795 break; 796 case '>': 797 append(">"); 798 break; 799 case '\"': 800 append("""); 801 break; 802 case '&': 803 /* 804 * Don't perform the quoting if this & is part of 805 * an mdoc escape sequence, e.g. \& 806 */ 807 if (src != ssrc && src[-1] != '\\') 808 append("&"); 809 else 810 append("&"); 811 break; 812 case '\002': 813 append("<b>"); 814 break; 815 case '\003': 816 append("</b>"); 817 break; 818 default: 819 *dst++ = *src; 820 break; 821 } 822 } 823 *dst = '\0'; 824 *slen = dst - ddst; 825 return ddst; 826 } 827 828 829 /* 830 * callback_html -- 831 * Callback function for run_query_html. It builds the html output and then 832 * calls the actual user supplied callback function. 833 */ 834 static int 835 callback_html(query_callback_args *callback_args) 836 { 837 struct orig_callback_data *orig_data = callback_args->other_data; 838 int (*callback)(query_callback_args*) = orig_data->callback; 839 size_t length = callback_args->snippet_length; 840 size_t name_description_length = strlen(callback_args->name_desc); 841 char *qsnippet = get_escaped_html_string(callback_args->snippet, &length); 842 char *qname_description = get_escaped_html_string(callback_args->name_desc, 843 &name_description_length); 844 callback_args->name_desc = qname_description; 845 callback_args->snippet = qsnippet; 846 callback_args->snippet_length = length; 847 callback_args->other_data = orig_data->data; 848 (*callback)(callback_args); 849 free(qsnippet); 850 free(qname_description); 851 return 0; 852 } 853 854 /* 855 * run_query_html -- 856 * Utility function to output query result in HTML format. 857 * It internally calls run_query only, but it first passes the output to its 858 * own custom callback function, which preprocess the snippet for quoting 859 * inline HTML fragments. 860 * After that it delegates the call the actual user supplied callback function. 861 */ 862 static int 863 run_query_html(sqlite3 *db, query_args *args) 864 { 865 struct orig_callback_data orig_data; 866 orig_data.callback = args->callback; 867 orig_data.data = args->callback_data; 868 const char *snippet_args[] = {"\002", "\003", "..."}; 869 args->callback = &callback_html; 870 args->callback_data = (void *) &orig_data; 871 return run_query_internal(db, snippet_args, args); 872 } 873 874 /* 875 * underline a string, pager style. 876 */ 877 static char * 878 ul_pager(int ul, const char *s) 879 { 880 size_t len; 881 char *dst, *d; 882 883 if (!ul) 884 return estrdup(s); 885 886 // a -> _\ba 887 len = strlen(s) * 3 + 1; 888 889 d = dst = emalloc(len); 890 while (*s) { 891 *d++ = '_'; 892 *d++ = '\b'; 893 *d++ = *s++; 894 } 895 *d = '\0'; 896 return dst; 897 } 898 899 /* 900 * callback_pager -- 901 * A callback similar to callback_html. It overstrikes the matching text in 902 * the snippet so that it appears emboldened when viewed using a pager like 903 * more or less. 904 */ 905 static int 906 callback_pager(query_callback_args *callback_args) 907 { 908 struct orig_callback_data *orig_data = callback_args->other_data; 909 char *psnippet; 910 const char *temp = callback_args->snippet; 911 int count = 0; 912 int i = 0, did; 913 size_t sz = 0; 914 size_t psnippet_length; 915 916 /* Count the number of bytes of matching text. For each of these 917 * bytes we will use 2 extra bytes to overstrike it so that it 918 * appears bold when viewed using a pager. 919 */ 920 while (*temp) { 921 sz = strcspn(temp, "\002\003"); 922 temp += sz; 923 if (*temp == '\003') { 924 count += 2 * (sz); 925 } 926 temp++; 927 } 928 929 psnippet_length = callback_args->snippet_length + count; 930 psnippet = emalloc(psnippet_length + 1); 931 932 /* Copy the bytes from snippet to psnippet: 933 * 1. Copy the bytes before \002 as it is. 934 * 2. The bytes after \002 need to be overstriked till we 935 * encounter \003. 936 * 3. To overstrike a byte 'A' we need to write 'A\bA' 937 */ 938 did = 0; 939 const char *snippet = callback_args->snippet; 940 while (*snippet) { 941 sz = strcspn(snippet, "\002"); 942 memcpy(&psnippet[i], snippet, sz); 943 snippet += sz; 944 i += sz; 945 946 /* Don't change this. Advancing the pointer without reading the byte 947 * is causing strange behavior. 948 */ 949 if (*snippet == '\002') 950 snippet++; 951 while (*snippet && *snippet != '\003') { 952 did = 1; 953 psnippet[i++] = *snippet; 954 psnippet[i++] = '\b'; 955 psnippet[i++] = *snippet++; 956 } 957 if (*snippet) 958 snippet++; 959 } 960 961 psnippet[i] = 0; 962 char *ul_section = ul_pager(did, callback_args->section); 963 char *ul_name = ul_pager(did, callback_args->name); 964 char *ul_name_desc = ul_pager(did, callback_args->name_desc); 965 callback_args->section = ul_section; 966 callback_args->name = ul_name; 967 callback_args->name_desc = ul_name_desc; 968 callback_args->snippet = psnippet; 969 callback_args->snippet_length = psnippet_length; 970 callback_args->other_data = orig_data->data; 971 (orig_data->callback)(callback_args); 972 free(ul_section); 973 free(ul_name); 974 free(ul_name_desc); 975 free(psnippet); 976 return 0; 977 } 978 979 struct term_args { 980 struct orig_callback_data *orig_data; 981 const char *smul; 982 const char *rmul; 983 }; 984 985 /* 986 * underline a string, pager style. 987 */ 988 static char * 989 ul_term(const char *s, const struct term_args *ta) 990 { 991 char *dst; 992 993 easprintf(&dst, "%s%s%s", ta->smul, s, ta->rmul); 994 return dst; 995 } 996 997 /* 998 * callback_term -- 999 * A callback similar to callback_html. It overstrikes the matching text in 1000 * the snippet so that it appears emboldened when viewed using a pager like 1001 * more or less. 1002 */ 1003 static int 1004 callback_term(query_callback_args *callback_args) 1005 { 1006 struct term_args *ta = callback_args->other_data; 1007 struct orig_callback_data *orig_data = ta->orig_data; 1008 1009 char *ul_section = ul_term(callback_args->section, ta); 1010 char *ul_name = ul_term(callback_args->name, ta); 1011 char *ul_name_desc = ul_term(callback_args->name_desc, ta); 1012 callback_args->section = ul_section; 1013 callback_args->name = ul_name; 1014 callback_args->name_desc = ul_name_desc; 1015 callback_args->other_data = orig_data->data; 1016 (orig_data->callback)(callback_args); 1017 free(ul_section); 1018 free(ul_name); 1019 free(ul_name_desc); 1020 return 0; 1021 } 1022 1023 /* 1024 * run_query_pager -- 1025 * Utility function similar to run_query_html. This function tries to 1026 * pre-process the result assuming it will be piped to a pager. 1027 * For this purpose it first calls its own callback function callback_pager 1028 * which then delegates the call to the user supplied callback. 1029 */ 1030 static int 1031 run_query_pager(sqlite3 *db, query_args *args) 1032 { 1033 struct orig_callback_data orig_data; 1034 orig_data.callback = args->callback; 1035 orig_data.data = args->callback_data; 1036 const char *snippet_args[3] = { "\002", "\003", "..." }; 1037 args->callback = &callback_pager; 1038 args->callback_data = (void *) &orig_data; 1039 return run_query_internal(db, snippet_args, args); 1040 } 1041 1042 struct nv { 1043 char *s; 1044 size_t l; 1045 }; 1046 1047 static int 1048 term_putc(int c, void *p) 1049 { 1050 struct nv *nv = p; 1051 nv->s[nv->l++] = c; 1052 return 0; 1053 } 1054 1055 static char * 1056 term_fix_seq(TERMINAL *ti, const char *seq) 1057 { 1058 char *res = estrdup(seq); 1059 struct nv nv; 1060 1061 if (ti == NULL) 1062 return res; 1063 1064 nv.s = res; 1065 nv.l = 0; 1066 ti_puts(ti, seq, 1, term_putc, &nv); 1067 nv.s[nv.l] = '\0'; 1068 1069 return res; 1070 } 1071 1072 static void 1073 term_init(int fd, const char *sa[5]) 1074 { 1075 TERMINAL *ti; 1076 int error; 1077 const char *bold, *sgr0, *smso, *rmso, *smul, *rmul; 1078 1079 if (ti_setupterm(&ti, NULL, fd, &error) == -1) { 1080 bold = sgr0 = NULL; 1081 smso = rmso = smul = rmul = ""; 1082 ti = NULL; 1083 } else { 1084 bold = ti_getstr(ti, "bold"); 1085 sgr0 = ti_getstr(ti, "sgr0"); 1086 if (bold == NULL || sgr0 == NULL) { 1087 smso = ti_getstr(ti, "smso"); 1088 1089 if (smso == NULL || 1090 (rmso = ti_getstr(ti, "rmso")) == NULL) 1091 smso = rmso = ""; 1092 bold = sgr0 = NULL; 1093 } else 1094 smso = rmso = ""; 1095 1096 smul = ti_getstr(ti, "smul"); 1097 if (smul == NULL || (rmul = ti_getstr(ti, "rmul")) == NULL) 1098 smul = rmul = ""; 1099 } 1100 1101 sa[0] = term_fix_seq(ti, bold ? bold : smso); 1102 sa[1] = term_fix_seq(ti, sgr0 ? sgr0 : rmso); 1103 sa[2] = estrdup("..."); 1104 sa[3] = term_fix_seq(ti, smul); 1105 sa[4] = term_fix_seq(ti, rmul); 1106 1107 if (ti) 1108 del_curterm(ti); 1109 } 1110 1111 /* 1112 * run_query_term -- 1113 * Utility function similar to run_query_html. This function tries to 1114 * pre-process the result assuming it will be displayed on a terminal 1115 * For this purpose it first calls its own callback function callback_pager 1116 * which then delegates the call to the user supplied callback. 1117 */ 1118 static int 1119 run_query_term(sqlite3 *db, query_args *args) 1120 { 1121 struct orig_callback_data orig_data; 1122 struct term_args ta; 1123 orig_data.callback = args->callback; 1124 orig_data.data = args->callback_data; 1125 const char *snippet_args[5]; 1126 1127 term_init(STDOUT_FILENO, snippet_args); 1128 ta.smul = snippet_args[3]; 1129 ta.rmul = snippet_args[4]; 1130 ta.orig_data = (void *) &orig_data; 1131 1132 args->callback = &callback_term; 1133 args->callback_data = &ta; 1134 return run_query_internal(db, snippet_args, args); 1135 } 1136 1137 static int 1138 run_query_none(sqlite3 *db, query_args *args) 1139 { 1140 struct orig_callback_data orig_data; 1141 orig_data.callback = args->callback; 1142 orig_data.data = args->callback_data; 1143 const char *snippet_args[3] = { "", "", "..." }; 1144 args->callback = &callback_pager; 1145 args->callback_data = (void *) &orig_data; 1146 return run_query_internal(db, snippet_args, args); 1147 } 1148 1149 int 1150 run_query(sqlite3 *db, query_format fmt, query_args *args) 1151 { 1152 switch (fmt) { 1153 case APROPOS_NONE: 1154 return run_query_none(db, args); 1155 case APROPOS_HTML: 1156 return run_query_html(db, args); 1157 case APROPOS_TERM: 1158 return run_query_term(db, args); 1159 case APROPOS_PAGER: 1160 return run_query_pager(db, args); 1161 default: 1162 warnx("Unknown query format %d", (int)fmt); 1163 return -1; 1164 } 1165 } 1166