1 /* $NetBSD: apropos-utils.c,v 1.39 2017/08/01 16:16:32 abhinav Exp $ */ 2 /*- 3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com> 4 * All rights reserved. 5 * 6 * This code was developed as part of Google's Summer of Code 2011 program. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __RCSID("$NetBSD: apropos-utils.c,v 1.39 2017/08/01 16:16:32 abhinav Exp $"); 35 36 #include <sys/queue.h> 37 #include <sys/stat.h> 38 39 #include <assert.h> 40 #include <ctype.h> 41 #include <err.h> 42 #include <math.h> 43 #include <stdio.h> 44 #include <stdlib.h> 45 #include <string.h> 46 #include <util.h> 47 #include <zlib.h> 48 #include <term.h> 49 #include <unistd.h> 50 #undef tab // XXX: manconf.h 51 52 #include "apropos-utils.h" 53 #include "custom_apropos_tokenizer.h" 54 #include "manconf.h" 55 #include "fts3_tokenizer.h" 56 57 typedef struct orig_callback_data { 58 void *data; 59 int (*callback) (void *, const char *, const char *, const char *, 60 const char *, size_t); 61 } orig_callback_data; 62 63 typedef struct inverse_document_frequency { 64 double value; 65 int status; 66 } inverse_document_frequency; 67 68 /* weights for individual columns */ 69 static const double col_weights[] = { 70 2.0, // NAME 71 2.00, // Name-description 72 0.55, // DESCRIPTION 73 0.10, // LIBRARY 74 0.001, //RETURN VALUES 75 0.20, //ENVIRONMENT 76 0.01, //FILES 77 0.001, //EXIT STATUS 78 2.00, //DIAGNOSTICS 79 0.05, //ERRORS 80 0.00, //md5_hash 81 1.00 //machine 82 }; 83 84 #ifndef APROPOS_DEBUG 85 static int 86 register_tokenizer(sqlite3 *db) 87 { 88 int rc; 89 sqlite3_stmt *stmt; 90 const sqlite3_tokenizer_module *p; 91 const char *name = "custom_apropos_tokenizer"; 92 get_custom_apropos_tokenizer(&p); 93 const char *sql = "SELECT fts3_tokenizer(?, ?)"; 94 95 sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); 96 rc = sqlite3_prepare_v2(db, sql, -1, &stmt, 0); 97 if (rc != SQLITE_OK) 98 return rc; 99 100 sqlite3_bind_text(stmt, 1, name, -1, SQLITE_STATIC); 101 sqlite3_bind_blob(stmt, 2, &p, sizeof(p), SQLITE_STATIC); 102 sqlite3_step(stmt); 103 104 return sqlite3_finalize(stmt); 105 } 106 #endif 107 108 /* 109 * lower -- 110 * Converts the string str to lower case 111 */ 112 char * 113 lower(char *str) 114 { 115 assert(str); 116 int i = 0; 117 char c; 118 while ((c = str[i]) != '\0') 119 str[i++] = tolower((unsigned char) c); 120 return str; 121 } 122 123 /* 124 * concat-- 125 * Utility function. Concatenates together: dst, a space character and src. 126 * dst + " " + src 127 */ 128 void 129 concat(char **dst, const char *src) 130 { 131 concat2(dst, src, strlen(src)); 132 } 133 134 void 135 concat2(char **dst, const char *src, size_t srclen) 136 { 137 size_t totallen, dstlen; 138 char *mydst = *dst; 139 assert(src != NULL); 140 141 /* 142 * If destination buffer dst is NULL, then simply 143 * strdup the source buffer 144 */ 145 if (mydst == NULL) { 146 mydst = estrndup(src, srclen); 147 *dst = mydst; 148 return; 149 } 150 151 dstlen = strlen(mydst); 152 /* 153 * NUL Byte and separator space 154 */ 155 totallen = dstlen + srclen + 2; 156 157 mydst = erealloc(mydst, totallen); 158 159 /* Append a space at the end of dst */ 160 mydst[dstlen++] = ' '; 161 162 /* Now, copy src at the end of dst */ 163 memcpy(mydst + dstlen, src, srclen); 164 mydst[dstlen + srclen] = '\0'; 165 *dst = mydst; 166 } 167 168 void 169 close_db(sqlite3 *db) 170 { 171 sqlite3_close(db); 172 sqlite3_shutdown(); 173 } 174 175 /* 176 * create_db -- 177 * Creates the database schema. 178 */ 179 static int 180 create_db(sqlite3 *db) 181 { 182 const char *sqlstr = NULL; 183 char *schemasql; 184 char *errmsg = NULL; 185 186 /*------------------------ Create the tables------------------------------*/ 187 188 #if NOTYET 189 sqlite3_exec(db, "PRAGMA journal_mode = WAL", NULL, NULL, NULL); 190 #else 191 sqlite3_exec(db, "PRAGMA journal_mode = DELETE", NULL, NULL, NULL); 192 #endif 193 194 schemasql = sqlite3_mprintf("PRAGMA user_version = %d", 195 APROPOS_SCHEMA_VERSION); 196 sqlite3_exec(db, schemasql, NULL, NULL, &errmsg); 197 if (errmsg != NULL) 198 goto out; 199 sqlite3_free(schemasql); 200 201 sqlstr = 202 //mandb 203 "CREATE VIRTUAL TABLE mandb USING fts4(section, name, " 204 "name_desc, desc, lib, return_vals, env, files, " 205 "exit_status, diagnostics, errors, md5_hash UNIQUE, machine, " 206 #ifndef APROPOS_DEBUG 207 "compress=zip, uncompress=unzip, tokenize=custom_apropos_tokenizer, " 208 #else 209 "tokenize=porter, " 210 #endif 211 "notindexed=section, notindexed=md5_hash); " 212 //mandb_meta 213 "CREATE TABLE IF NOT EXISTS mandb_meta(device, inode, mtime, " 214 "file UNIQUE, md5_hash UNIQUE, id INTEGER PRIMARY KEY); " 215 //mandb_links 216 "CREATE TABLE IF NOT EXISTS mandb_links(link COLLATE NOCASE, target, section, " 217 "machine, md5_hash); "; 218 219 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 220 if (errmsg != NULL) 221 goto out; 222 223 sqlstr = 224 "CREATE INDEX IF NOT EXISTS index_mandb_links ON mandb_links " 225 "(link); " 226 "CREATE INDEX IF NOT EXISTS index_mandb_meta_dev ON mandb_meta " 227 "(device, inode); " 228 "CREATE INDEX IF NOT EXISTS index_mandb_links_md5 ON mandb_links " 229 "(md5_hash);"; 230 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 231 if (errmsg != NULL) 232 goto out; 233 return 0; 234 235 out: 236 warnx("%s", errmsg); 237 free(errmsg); 238 sqlite3_close(db); 239 sqlite3_shutdown(); 240 return -1; 241 } 242 243 /* 244 * zip -- 245 * User defined Sqlite function to compress the FTS table 246 */ 247 static void 248 zip(sqlite3_context *pctx, int nval, sqlite3_value **apval) 249 { 250 int nin; 251 long int nout; 252 const unsigned char * inbuf; 253 unsigned char *outbuf; 254 255 assert(nval == 1); 256 nin = sqlite3_value_bytes(apval[0]); 257 inbuf = (const unsigned char *) sqlite3_value_blob(apval[0]); 258 nout = nin + 13 + (nin + 999) / 1000; 259 outbuf = emalloc(nout); 260 compress(outbuf, (unsigned long *) &nout, inbuf, nin); 261 sqlite3_result_blob(pctx, outbuf, nout, free); 262 } 263 264 /* 265 * unzip -- 266 * User defined Sqlite function to uncompress the FTS table. 267 */ 268 static void 269 unzip(sqlite3_context *pctx, int nval, sqlite3_value **apval) 270 { 271 unsigned int rc; 272 unsigned char *outbuf; 273 z_stream stream; 274 275 assert(nval == 1); 276 stream.next_in = __UNCONST(sqlite3_value_blob(apval[0])); 277 stream.avail_in = sqlite3_value_bytes(apval[0]); 278 stream.avail_out = stream.avail_in * 2 + 100; 279 stream.next_out = outbuf = emalloc(stream.avail_out); 280 stream.zalloc = NULL; 281 stream.zfree = NULL; 282 283 if (inflateInit(&stream) != Z_OK) { 284 free(outbuf); 285 return; 286 } 287 288 while ((rc = inflate(&stream, Z_SYNC_FLUSH)) != Z_STREAM_END) { 289 if (rc != Z_OK || 290 (stream.avail_out != 0 && stream.avail_in == 0)) { 291 free(outbuf); 292 return; 293 } 294 outbuf = erealloc(outbuf, stream.total_out * 2); 295 stream.next_out = outbuf + stream.total_out; 296 stream.avail_out = stream.total_out; 297 } 298 if (inflateEnd(&stream) != Z_OK) { 299 free(outbuf); 300 return; 301 } 302 outbuf = erealloc(outbuf, stream.total_out); 303 sqlite3_result_text(pctx, (const char *)outbuf, stream.total_out, free); 304 } 305 306 /* 307 * get_dbpath -- 308 * Read the path of the database from man.conf and return. 309 */ 310 char * 311 get_dbpath(const char *manconf) 312 { 313 TAG *tp; 314 char *dbpath; 315 316 config(manconf); 317 tp = gettag("_mandb", 1); 318 if (!tp) 319 return NULL; 320 321 if (TAILQ_EMPTY(&tp->entrylist)) 322 return NULL; 323 324 dbpath = TAILQ_LAST(&tp->entrylist, tqh)->s; 325 return dbpath; 326 } 327 328 /* init_db -- 329 * Prepare the database. Register the compress/uncompress functions and the 330 * stopword tokenizer. 331 * db_flag specifies the mode in which to open the database. 3 options are 332 * available: 333 * 1. DB_READONLY: Open in READONLY mode. An error if db does not exist. 334 * 2. DB_READWRITE: Open in read-write mode. An error if db does not exist. 335 * 3. DB_CREATE: Open in read-write mode. It will try to create the db if 336 * it does not exist already. 337 * RETURN VALUES: 338 * The function will return NULL in case the db does not exist 339 * and DB_CREATE 340 * was not specified. And in case DB_CREATE was specified and yet NULL is 341 * returned, then there was some other error. 342 * In normal cases the function should return a handle to the db. 343 */ 344 sqlite3 * 345 init_db(mandb_access_mode db_flag, const char *manconf) 346 { 347 sqlite3 *db = NULL; 348 sqlite3_stmt *stmt; 349 struct stat sb; 350 int rc; 351 int create_db_flag = 0; 352 353 char *dbpath = get_dbpath(manconf); 354 if (dbpath == NULL) 355 errx(EXIT_FAILURE, "_mandb entry not found in man.conf"); 356 357 if (!(stat(dbpath, &sb) == 0 && S_ISREG(sb.st_mode))) { 358 /* Database does not exist, check if DB_CREATE was specified, 359 * and set flag to create the database schema 360 */ 361 if (db_flag != (MANDB_CREATE)) { 362 warnx("Missing apropos database. " 363 "Please run makemandb to create it."); 364 return NULL; 365 } 366 create_db_flag = 1; 367 } else { 368 /* 369 * Database exists. Check if we have the permissions 370 * to read/write the files 371 */ 372 int access_mode = R_OK; 373 switch (db_flag) { 374 case MANDB_CREATE: 375 case MANDB_WRITE: 376 access_mode |= W_OK; 377 break; 378 default: 379 break; 380 } 381 if ((access(dbpath, access_mode)) != 0) { 382 warnx("Unable to access the database, please check" 383 " permissions for `%s'", dbpath); 384 return NULL; 385 } 386 } 387 388 sqlite3_initialize(); 389 rc = sqlite3_open_v2(dbpath, &db, db_flag, NULL); 390 391 if (rc != SQLITE_OK) { 392 warnx("%s", sqlite3_errmsg(db)); 393 goto error; 394 } 395 396 sqlite3_extended_result_codes(db, 1); 397 398 #ifndef APROPOS_DEBUG 399 rc = register_tokenizer(db); 400 if (rc != SQLITE_OK) { 401 warnx("Unable to register custom tokenizer: %s", sqlite3_errmsg(db)); 402 goto error; 403 } 404 #endif 405 406 if (create_db_flag && create_db(db) < 0) { 407 warnx("%s", "Unable to create database schema"); 408 goto error; 409 } 410 411 rc = sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &stmt, NULL); 412 if (rc != SQLITE_OK) { 413 warnx("Unable to query schema version: %s", 414 sqlite3_errmsg(db)); 415 goto error; 416 } 417 if (sqlite3_step(stmt) != SQLITE_ROW) { 418 sqlite3_finalize(stmt); 419 warnx("Unable to query schema version: %s", 420 sqlite3_errmsg(db)); 421 goto error; 422 } 423 if (sqlite3_column_int(stmt, 0) != APROPOS_SCHEMA_VERSION) { 424 sqlite3_finalize(stmt); 425 warnx("Incorrect schema version found. " 426 "Please run makemandb -f."); 427 goto error; 428 } 429 sqlite3_finalize(stmt); 430 431 432 /* Register the zip and unzip functions for FTS compression */ 433 rc = sqlite3_create_function(db, "zip", 1, SQLITE_ANY, NULL, zip, 434 NULL, NULL); 435 if (rc != SQLITE_OK) { 436 warnx("Unable to register function: compress: %s", 437 sqlite3_errmsg(db)); 438 goto error; 439 } 440 441 rc = sqlite3_create_function(db, "unzip", 1, SQLITE_ANY, NULL, 442 unzip, NULL, NULL); 443 if (rc != SQLITE_OK) { 444 warnx("Unable to register function: uncompress: %s", 445 sqlite3_errmsg(db)); 446 goto error; 447 } 448 return db; 449 450 error: 451 close_db(db); 452 return NULL; 453 } 454 455 /* 456 * rank_func -- 457 * Sqlite user defined function for ranking the documents. 458 * For each phrase of the query, it computes the tf and idf and adds them over. 459 * It computes the final rank, by multiplying tf and idf together. 460 * Weight of term t for document d = (term frequency of t in d * 461 * inverse document frequency of t) 462 * 463 * Term Frequency of term t in document d = Number of times t occurs in d / 464 * Number of times t appears in all documents 465 * 466 * Inverse document frequency of t = log(Total number of documents / 467 * Number of documents in which t occurs) 468 */ 469 static void 470 rank_func(sqlite3_context *pctx, int nval, sqlite3_value **apval) 471 { 472 inverse_document_frequency *idf = sqlite3_user_data(pctx); 473 double tf = 0.0; 474 const unsigned int *matchinfo; 475 int ncol; 476 int nphrase; 477 int iphrase; 478 int ndoc; 479 int doclen = 0; 480 const double k = 3.75; 481 /* 482 * Check that the number of arguments passed to this 483 * function is correct. 484 */ 485 assert(nval == 1); 486 487 matchinfo = (const unsigned int *) sqlite3_value_blob(apval[0]); 488 nphrase = matchinfo[0]; 489 ncol = matchinfo[1]; 490 ndoc = matchinfo[2 + 3 * ncol * nphrase + ncol]; 491 for (iphrase = 0; iphrase < nphrase; iphrase++) { 492 int icol; 493 const unsigned int *phraseinfo = 494 &matchinfo[2 + ncol + iphrase * ncol * 3]; 495 for(icol = 1; icol < ncol; icol++) { 496 497 /* nhitcount: number of times the current phrase occurs 498 * in the current column in the current document. 499 * nglobalhitcount: number of times current phrase 500 * occurs in the current column in all documents. 501 * ndocshitcount: number of documents in which the 502 * current phrase occurs in the current column at 503 * least once. 504 */ 505 int nhitcount = phraseinfo[3 * icol]; 506 int nglobalhitcount = phraseinfo[3 * icol + 1]; 507 int ndocshitcount = phraseinfo[3 * icol + 2]; 508 doclen = matchinfo[2 + icol ]; 509 double weight = col_weights[icol - 1]; 510 if (idf->status == 0 && ndocshitcount) 511 idf->value += 512 log(((double)ndoc / ndocshitcount))* weight; 513 514 /* 515 * Dividing the tf by document length to normalize 516 * the effect of longer documents. 517 */ 518 if (nglobalhitcount > 0 && nhitcount) 519 tf += (((double)nhitcount * weight) 520 / (nglobalhitcount * doclen)); 521 } 522 } 523 idf->status = 1; 524 525 /* 526 * Final score: Dividing by k + tf further normalizes the weight 527 * leading to better results. The value of k is experimental 528 */ 529 double score = (tf * idf->value) / (k + tf); 530 sqlite3_result_double(pctx, score); 531 return; 532 } 533 534 /* 535 * generates sql query for matching the user entered query 536 */ 537 static char * 538 generate_search_query(query_args *args, const char *snippet_args[3]) 539 { 540 const char *default_snippet_args[3]; 541 char *section_clause = NULL; 542 char *limit_clause = NULL; 543 char *machine_clause = NULL; 544 char *query = NULL; 545 546 if (args->machine) { 547 machine_clause = sqlite3_mprintf("AND mandb.machine=%Q", args->machine); 548 if (machine_clause == NULL) 549 goto RETURN; 550 } 551 552 if (args->nrec >= 0) { 553 /* Use the provided number of records and offset */ 554 limit_clause = sqlite3_mprintf(" LIMIT %d OFFSET %d", 555 args->nrec, args->offset); 556 if (limit_clause == NULL) 557 goto RETURN; 558 } 559 560 /* We want to build a query of the form: "select x,y,z from mandb where 561 * mandb match :query [AND (section IN ('1', '2')] 562 * ORDER BY rank DESC [LIMIT 10 OFFSET 0]" 563 * NOTES: 564 * 1. The portion in first pair of square brackets is optional. 565 * It will be there only if the user has specified an option 566 * to search in one or more specific sections. 567 * 2. The LIMIT portion will be there if the user has specified 568 * a limit using the -n option. 569 */ 570 if (args->sections && args->sections[0]) { 571 concat(§ion_clause, " AND mandb.section IN ("); 572 for (size_t i = 0; args->sections[i]; i++) { 573 char *temp; 574 char c = args->sections[i + 1]? ',': ')'; 575 if ((temp = sqlite3_mprintf("%Q%c", args->sections[i], c)) == NULL) 576 goto RETURN; 577 concat(§ion_clause, temp); 578 free(temp); 579 } 580 } 581 582 if (snippet_args == NULL) { 583 default_snippet_args[0] = ""; 584 default_snippet_args[1] = ""; 585 default_snippet_args[2] = "..."; 586 snippet_args = default_snippet_args; 587 } 588 589 if (args->legacy) { 590 char *wild; 591 easprintf(&wild, "%%%s%%", args->search_str); 592 query = sqlite3_mprintf("SELECT section, name, name_desc, machine" 593 " FROM mandb" 594 " WHERE name LIKE %Q OR name_desc LIKE %Q " 595 "%s" 596 "%s", 597 wild, wild, 598 section_clause ? section_clause : "", 599 limit_clause ? limit_clause : ""); 600 free(wild); 601 } else if (strchr(args->search_str, ' ') == NULL) { 602 /* 603 * If it's a single word query, we want to search in the 604 * links table as well. If the link table contains an entry 605 * for the queried keyword, we want to use that as the name of 606 * the man page. 607 * For example, for `apropos realloc` the output should be 608 * realloc(3) and not malloc(3). 609 */ 610 query = sqlite3_mprintf( 611 "SELECT section, name, name_desc, machine," 612 " snippet(mandb, %Q, %Q, %Q, -1, 40 )," 613 " rank_func(matchinfo(mandb, \"pclxn\")) AS rank" 614 " FROM mandb WHERE name NOT IN (" 615 " SELECT target FROM mandb_links WHERE link=%Q AND" 616 " mandb_links.section=mandb.section) AND mandb MATCH %Q %s %s" 617 " UNION" 618 " SELECT mandb.section, mandb_links.link AS name, mandb.name_desc," 619 " mandb.machine, '' AS snippet, 100.00 AS rank" 620 " FROM mandb JOIN mandb_links ON mandb.name=mandb_links.target and" 621 " mandb.section=mandb_links.section WHERE mandb_links.link=%Q" 622 " %s %s" 623 " ORDER BY rank DESC %s", 624 snippet_args[0], snippet_args[1], snippet_args[2], 625 args->search_str, args->search_str, section_clause ? section_clause : "", 626 machine_clause ? machine_clause : "", args->search_str, 627 machine_clause ? machine_clause : "", 628 section_clause ? section_clause : "", 629 limit_clause ? limit_clause : ""); 630 } else { 631 query = sqlite3_mprintf("SELECT section, name, name_desc, machine," 632 " snippet(mandb, %Q, %Q, %Q, -1, 40 )," 633 " rank_func(matchinfo(mandb, \"pclxn\")) AS rank" 634 " FROM mandb" 635 " WHERE mandb MATCH %Q %s " 636 "%s" 637 " ORDER BY rank DESC" 638 "%s", 639 snippet_args[0], snippet_args[1], snippet_args[2], 640 args->search_str, machine_clause ? machine_clause : "", 641 section_clause ? section_clause : "", 642 limit_clause ? limit_clause : ""); 643 } 644 645 RETURN: 646 free(machine_clause); 647 free(section_clause); 648 free(limit_clause); 649 return query; 650 } 651 652 /* 653 * Execute the full text search query and return the number of results 654 * obtained. 655 */ 656 static unsigned int 657 execute_search_query(sqlite3 *db, char *query, query_args *args) 658 { 659 sqlite3_stmt *stmt; 660 const char *section; 661 char *name; 662 char *slash_ptr; 663 const char *name_desc; 664 const char *machine; 665 const char *snippet = ""; 666 const char *name_temp; 667 char *m = NULL; 668 int rc; 669 inverse_document_frequency idf = {0, 0}; 670 671 if (!args->legacy) { 672 /* Register the rank function */ 673 rc = sqlite3_create_function(db, "rank_func", 1, SQLITE_ANY, 674 (void *) &idf, rank_func, NULL, NULL); 675 if (rc != SQLITE_OK) { 676 warnx("Unable to register the ranking function: %s", 677 sqlite3_errmsg(db)); 678 sqlite3_close(db); 679 sqlite3_shutdown(); 680 exit(EXIT_FAILURE); 681 } 682 } 683 684 rc = sqlite3_prepare_v2(db, query, -1, &stmt, NULL); 685 if (rc == SQLITE_IOERR) { 686 warnx("Corrupt database. Please rerun makemandb"); 687 return -1; 688 } else if (rc != SQLITE_OK) { 689 warnx("%s", sqlite3_errmsg(db)); 690 return -1; 691 } 692 693 unsigned int nresults = 0; 694 while (sqlite3_step(stmt) == SQLITE_ROW) { 695 nresults++; 696 section = (const char *) sqlite3_column_text(stmt, 0); 697 name_temp = (const char *) sqlite3_column_text(stmt, 1); 698 name_desc = (const char *) sqlite3_column_text(stmt, 2); 699 machine = (const char *) sqlite3_column_text(stmt, 3); 700 if (!args->legacy) 701 snippet = (const char *) sqlite3_column_text(stmt, 4); 702 if ((slash_ptr = strrchr(name_temp, '/')) != NULL) 703 name_temp = slash_ptr + 1; 704 if (machine && machine[0]) { 705 m = estrdup(machine); 706 easprintf(&name, "%s/%s", lower(m), name_temp); 707 free(m); 708 } else { 709 name = estrdup((const char *) 710 sqlite3_column_text(stmt, 1)); 711 } 712 713 (args->callback)(args->callback_data, section, name, 714 name_desc, snippet, args->legacy? 0: strlen(snippet)); 715 free(name); 716 } 717 sqlite3_finalize(stmt); 718 return nresults; 719 } 720 721 722 /* 723 * run_query_internal -- 724 * Performs the searches for the keywords entered by the user. 725 * The 2nd param: snippet_args is an array of strings providing values for the 726 * last three parameters to the snippet function of sqlite. (Look at the docs). 727 * The 3rd param: args contains rest of the search parameters. Look at 728 * arpopos-utils.h for the description of individual fields. 729 * 730 */ 731 static int 732 run_query_internal(sqlite3 *db, const char *snippet_args[3], query_args *args) 733 { 734 char *query; 735 query = generate_search_query(args, snippet_args); 736 if (query == NULL) { 737 *args->errmsg = estrdup("malloc failed"); 738 return -1; 739 } 740 741 execute_search_query(db, query, args); 742 sqlite3_free(query); 743 return *(args->errmsg) == NULL ? 0 : -1; 744 } 745 746 static char * 747 get_escaped_html_string(const char *src, size_t *slen) 748 { 749 static const char trouble[] = "<>\"&\002\003"; 750 /* 751 * First scan the src to find out the number of occurrences 752 * of {'>', '<' '"', '&'}. Then allocate a new buffer with 753 * sufficient space to be able to store the quoted versions 754 * of the special characters {>, <, ", &}. 755 * Copy over the characters from the original src into 756 * this buffer while replacing the special characters with 757 * their quoted versions. 758 */ 759 char *dst, *ddst; 760 size_t count; 761 const char *ssrc; 762 763 for (count = 0, ssrc = src; *src; count++) { 764 size_t sz = strcspn(src, trouble); 765 src += sz + 1; 766 } 767 768 769 #define append(a) \ 770 do { \ 771 memcpy(dst, (a), sizeof(a) - 1); \ 772 dst += sizeof(a) - 1; \ 773 } while (/*CONSTCOND*/0) 774 775 776 ddst = dst = emalloc(*slen + count * 5 + 1); 777 for (src = ssrc; *src; src++) { 778 switch (*src) { 779 case '<': 780 append("<"); 781 break; 782 case '>': 783 append(">"); 784 break; 785 case '\"': 786 append("""); 787 break; 788 case '&': 789 /* 790 * Don't perform the quoting if this & is part of 791 * an mdoc escape sequence, e.g. \& 792 */ 793 if (src != ssrc && src[-1] != '\\') 794 append("&"); 795 else 796 append("&"); 797 break; 798 case '\002': 799 append("<b>"); 800 break; 801 case '\003': 802 append("</b>"); 803 break; 804 default: 805 *dst++ = *src; 806 break; 807 } 808 } 809 *dst = '\0'; 810 *slen = dst - ddst; 811 return ddst; 812 } 813 814 815 /* 816 * callback_html -- 817 * Callback function for run_query_html. It builds the html output and then 818 * calls the actual user supplied callback function. 819 */ 820 static int 821 callback_html(void *data, const char *section, const char *name, 822 const char *name_desc, const char *snippet, size_t snippet_length) 823 { 824 struct orig_callback_data *orig_data = data; 825 int (*callback)(void *, const char *, const char *, const char *, 826 const char *, size_t) = orig_data->callback; 827 size_t length = snippet_length; 828 size_t name_description_length = strlen(name_desc); 829 char *qsnippet = get_escaped_html_string(snippet, &length); 830 char *qname_description = get_escaped_html_string(name_desc, 831 &name_description_length); 832 833 (*callback)(orig_data->data, section, name, qname_description, 834 qsnippet, length); 835 free(qsnippet); 836 free(qname_description); 837 return 0; 838 } 839 840 /* 841 * run_query_html -- 842 * Utility function to output query result in HTML format. 843 * It internally calls run_query only, but it first passes the output to its 844 * own custom callback function, which preprocess the snippet for quoting 845 * inline HTML fragments. 846 * After that it delegates the call the actual user supplied callback function. 847 */ 848 static int 849 run_query_html(sqlite3 *db, query_args *args) 850 { 851 struct orig_callback_data orig_data; 852 orig_data.callback = args->callback; 853 orig_data.data = args->callback_data; 854 const char *snippet_args[] = {"\002", "\003", "..."}; 855 args->callback = &callback_html; 856 args->callback_data = (void *) &orig_data; 857 return run_query_internal(db, snippet_args, args); 858 } 859 860 /* 861 * underline a string, pager style. 862 */ 863 static char * 864 ul_pager(int ul, const char *s) 865 { 866 size_t len; 867 char *dst, *d; 868 869 if (!ul) 870 return estrdup(s); 871 872 // a -> _\ba 873 len = strlen(s) * 3 + 1; 874 875 d = dst = emalloc(len); 876 while (*s) { 877 *d++ = '_'; 878 *d++ = '\b'; 879 *d++ = *s++; 880 } 881 *d = '\0'; 882 return dst; 883 } 884 885 /* 886 * callback_pager -- 887 * A callback similar to callback_html. It overstrikes the matching text in 888 * the snippet so that it appears emboldened when viewed using a pager like 889 * more or less. 890 */ 891 static int 892 callback_pager(void *data, const char *section, const char *name, 893 const char *name_desc, const char *snippet, size_t snippet_length) 894 { 895 struct orig_callback_data *orig_data = data; 896 char *psnippet; 897 const char *temp = snippet; 898 int count = 0; 899 int i = 0, did; 900 size_t sz = 0; 901 size_t psnippet_length; 902 903 /* Count the number of bytes of matching text. For each of these 904 * bytes we will use 2 extra bytes to overstrike it so that it 905 * appears bold when viewed using a pager. 906 */ 907 while (*temp) { 908 sz = strcspn(temp, "\002\003"); 909 temp += sz; 910 if (*temp == '\003') { 911 count += 2 * (sz); 912 } 913 temp++; 914 } 915 916 psnippet_length = snippet_length + count; 917 psnippet = emalloc(psnippet_length + 1); 918 919 /* Copy the bytes from snippet to psnippet: 920 * 1. Copy the bytes before \002 as it is. 921 * 2. The bytes after \002 need to be overstriked till we 922 * encounter \003. 923 * 3. To overstrike a byte 'A' we need to write 'A\bA' 924 */ 925 did = 0; 926 while (*snippet) { 927 sz = strcspn(snippet, "\002"); 928 memcpy(&psnippet[i], snippet, sz); 929 snippet += sz; 930 i += sz; 931 932 /* Don't change this. Advancing the pointer without reading the byte 933 * is causing strange behavior. 934 */ 935 if (*snippet == '\002') 936 snippet++; 937 while (*snippet && *snippet != '\003') { 938 did = 1; 939 psnippet[i++] = *snippet; 940 psnippet[i++] = '\b'; 941 psnippet[i++] = *snippet++; 942 } 943 if (*snippet) 944 snippet++; 945 } 946 947 psnippet[i] = 0; 948 char *ul_section = ul_pager(did, section); 949 char *ul_name = ul_pager(did, name); 950 char *ul_name_desc = ul_pager(did, name_desc); 951 (orig_data->callback)(orig_data->data, ul_section, ul_name, 952 ul_name_desc, psnippet, psnippet_length); 953 free(ul_section); 954 free(ul_name); 955 free(ul_name_desc); 956 free(psnippet); 957 return 0; 958 } 959 960 struct term_args { 961 struct orig_callback_data *orig_data; 962 const char *smul; 963 const char *rmul; 964 }; 965 966 /* 967 * underline a string, pager style. 968 */ 969 static char * 970 ul_term(const char *s, const struct term_args *ta) 971 { 972 char *dst; 973 974 easprintf(&dst, "%s%s%s", ta->smul, s, ta->rmul); 975 return dst; 976 } 977 978 /* 979 * callback_term -- 980 * A callback similar to callback_html. It overstrikes the matching text in 981 * the snippet so that it appears emboldened when viewed using a pager like 982 * more or less. 983 */ 984 static int 985 callback_term(void *data, const char *section, const char *name, 986 const char *name_desc, const char *snippet, size_t snippet_length) 987 { 988 struct term_args *ta = data; 989 struct orig_callback_data *orig_data = ta->orig_data; 990 991 char *ul_section = ul_term(section, ta); 992 char *ul_name = ul_term(name, ta); 993 char *ul_name_desc = ul_term(name_desc, ta); 994 (orig_data->callback)(orig_data->data, ul_section, ul_name, 995 ul_name_desc, snippet, snippet_length); 996 free(ul_section); 997 free(ul_name); 998 free(ul_name_desc); 999 return 0; 1000 } 1001 1002 /* 1003 * run_query_pager -- 1004 * Utility function similar to run_query_html. This function tries to 1005 * pre-process the result assuming it will be piped to a pager. 1006 * For this purpose it first calls its own callback function callback_pager 1007 * which then delegates the call to the user supplied callback. 1008 */ 1009 static int 1010 run_query_pager(sqlite3 *db, query_args *args) 1011 { 1012 struct orig_callback_data orig_data; 1013 orig_data.callback = args->callback; 1014 orig_data.data = args->callback_data; 1015 const char *snippet_args[3] = { "\002", "\003", "..." }; 1016 args->callback = &callback_pager; 1017 args->callback_data = (void *) &orig_data; 1018 return run_query_internal(db, snippet_args, args); 1019 } 1020 1021 struct nv { 1022 char *s; 1023 size_t l; 1024 }; 1025 1026 static int 1027 term_putc(int c, void *p) 1028 { 1029 struct nv *nv = p; 1030 nv->s[nv->l++] = c; 1031 return 0; 1032 } 1033 1034 static char * 1035 term_fix_seq(TERMINAL *ti, const char *seq) 1036 { 1037 char *res = estrdup(seq); 1038 struct nv nv; 1039 1040 if (ti == NULL) 1041 return res; 1042 1043 nv.s = res; 1044 nv.l = 0; 1045 ti_puts(ti, seq, 1, term_putc, &nv); 1046 nv.s[nv.l] = '\0'; 1047 1048 return res; 1049 } 1050 1051 static void 1052 term_init(int fd, const char *sa[5]) 1053 { 1054 TERMINAL *ti; 1055 int error; 1056 const char *bold, *sgr0, *smso, *rmso, *smul, *rmul; 1057 1058 if (ti_setupterm(&ti, NULL, fd, &error) == -1) { 1059 bold = sgr0 = NULL; 1060 smso = rmso = smul = rmul = ""; 1061 ti = NULL; 1062 } else { 1063 bold = ti_getstr(ti, "bold"); 1064 sgr0 = ti_getstr(ti, "sgr0"); 1065 if (bold == NULL || sgr0 == NULL) { 1066 smso = ti_getstr(ti, "smso"); 1067 1068 if (smso == NULL || 1069 (rmso = ti_getstr(ti, "rmso")) == NULL) 1070 smso = rmso = ""; 1071 bold = sgr0 = NULL; 1072 } else 1073 smso = rmso = ""; 1074 1075 smul = ti_getstr(ti, "smul"); 1076 if (smul == NULL || (rmul = ti_getstr(ti, "rmul")) == NULL) 1077 smul = rmul = ""; 1078 } 1079 1080 sa[0] = term_fix_seq(ti, bold ? bold : smso); 1081 sa[1] = term_fix_seq(ti, sgr0 ? sgr0 : rmso); 1082 sa[2] = estrdup("..."); 1083 sa[3] = term_fix_seq(ti, smul); 1084 sa[4] = term_fix_seq(ti, rmul); 1085 1086 if (ti) 1087 del_curterm(ti); 1088 } 1089 1090 /* 1091 * run_query_term -- 1092 * Utility function similar to run_query_html. This function tries to 1093 * pre-process the result assuming it will be displayed on a terminal 1094 * For this purpose it first calls its own callback function callback_pager 1095 * which then delegates the call to the user supplied callback. 1096 */ 1097 static int 1098 run_query_term(sqlite3 *db, query_args *args) 1099 { 1100 struct orig_callback_data orig_data; 1101 struct term_args ta; 1102 orig_data.callback = args->callback; 1103 orig_data.data = args->callback_data; 1104 const char *snippet_args[5]; 1105 1106 term_init(STDOUT_FILENO, snippet_args); 1107 ta.smul = snippet_args[3]; 1108 ta.rmul = snippet_args[4]; 1109 ta.orig_data = (void *) &orig_data; 1110 1111 args->callback = &callback_term; 1112 args->callback_data = &ta; 1113 return run_query_internal(db, snippet_args, args); 1114 } 1115 1116 static int 1117 run_query_none(sqlite3 *db, query_args *args) 1118 { 1119 struct orig_callback_data orig_data; 1120 orig_data.callback = args->callback; 1121 orig_data.data = args->callback_data; 1122 const char *snippet_args[3] = { "", "", "..." }; 1123 args->callback = &callback_pager; 1124 args->callback_data = (void *) &orig_data; 1125 return run_query_internal(db, snippet_args, args); 1126 } 1127 1128 int 1129 run_query(sqlite3 *db, query_format fmt, query_args *args) 1130 { 1131 switch (fmt) { 1132 case APROPOS_NONE: 1133 return run_query_none(db, args); 1134 case APROPOS_HTML: 1135 return run_query_html(db, args); 1136 case APROPOS_TERM: 1137 return run_query_term(db, args); 1138 case APROPOS_PAGER: 1139 return run_query_pager(db, args); 1140 default: 1141 warnx("Unknown query format %d", (int)fmt); 1142 return -1; 1143 } 1144 } 1145