1 /********************************************************************** 2 Copyright(c) 2011-2017 Intel Corporation All rights reserved. 3 4 Redistribution and use in source and binary forms, with or without 5 modification, are permitted provided that the following conditions 6 are met: 7 * Redistributions of source code must retain the above copyright 8 notice, this list of conditions and the following disclaimer. 9 * Redistributions in binary form must reproduce the above copyright 10 notice, this list of conditions and the following disclaimer in 11 the documentation and/or other materials provided with the 12 distribution. 13 * Neither the name of Intel Corporation nor the names of its 14 contributors may be used to endorse or promote products derived 15 from this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 **********************************************************************/ 29 30 #include <stdlib.h> 31 #include <stdint.h> 32 #include <stdarg.h> 33 #include <stdio.h> 34 #include <assert.h> 35 #include "isal_crypto_api.h" 36 #include "rolling_hashx.h" 37 #include "sha256_mb.h" 38 #include "test.h" 39 40 #define MAX_BUFFER_SIZE 128 * 1024 * 1024 41 #define HASH_POOL_SIZE SHA256_MAX_LANES 42 43 #ifndef TEST_SEED 44 #define TEST_SEED 0x1234 45 #endif 46 47 #define FILTER_BITS 10 48 #define FILTER_SIZE (1 << FILTER_BITS) 49 #define FILTER_MASK (FILTER_SIZE - 1) 50 51 #define BITS_TO_INDEX_LONG 6 52 #define MASK_TO_INDEX_LONG ((1 << BITS_TO_INDEX_LONG) - 1) 53 54 // Globals 55 SHA256_HASH_CTX ctxpool[SHA256_MAX_LANES], *last_ctx; 56 SHA256_HASH_CTX_MGR mb_hash_mgr; 57 uint64_t filter_table[FILTER_SIZE]; 58 unsigned long chunks_created = 0; 59 unsigned long filter_hits = 0; 60 61 // Example function to run on each chunk 62 63 void 64 run_fragment(SHA256_HASH_CTX *ctx) 65 { 66 uint64_t lookup, set_hash; 67 unsigned int lookup_hash; 68 uint32_t idx; 69 70 chunks_created++; 71 72 // Run a simple lookup filter on chunk using digest 73 lookup_hash = ctx->job.result_digest[0] & FILTER_MASK; 74 lookup = filter_table[lookup_hash]; 75 76 idx = ctx->job.result_digest[1]; 77 78 set_hash = 1 << (idx & MASK_TO_INDEX_LONG) | 79 1 << ((idx >> BITS_TO_INDEX_LONG) & MASK_TO_INDEX_LONG) | 80 1 << ((idx >> (2 * BITS_TO_INDEX_LONG)) & MASK_TO_INDEX_LONG); 81 82 if ((lookup & set_hash) == set_hash) 83 filter_hits++; 84 else 85 filter_table[lookup_hash] = lookup | set_hash; 86 } 87 88 void 89 setup_chunk_processing(void) 90 { 91 int i; 92 93 sha256_ctx_mgr_init(&mb_hash_mgr); 94 95 for (i = 0; i < HASH_POOL_SIZE; i++) 96 hash_ctx_init(&ctxpool[i]); 97 98 last_ctx = &ctxpool[0]; 99 } 100 101 SHA256_HASH_CTX * 102 get_next_job_ctx(void) 103 { 104 int i; 105 SHA256_HASH_CTX *ctx; 106 107 if (last_ctx && hash_ctx_complete(last_ctx)) 108 return last_ctx; 109 110 for (i = 0; i < HASH_POOL_SIZE; i++) { 111 if (hash_ctx_complete(&ctxpool[i])) 112 return &ctxpool[i]; 113 } 114 ctx = sha256_ctx_mgr_flush(&mb_hash_mgr); 115 assert(ctx != NULL); 116 return ctx; 117 } 118 119 void 120 put_next_job_ctx(SHA256_HASH_CTX *ctx) 121 { 122 if (ctx && hash_ctx_complete(ctx)) 123 last_ctx = ctx; 124 125 run_fragment(ctx); 126 } 127 128 void 129 process_chunk(uint8_t *buff, int len) 130 { 131 SHA256_HASH_CTX *ctx; 132 133 ctx = get_next_job_ctx(); 134 ctx = sha256_ctx_mgr_submit(&mb_hash_mgr, ctx, buff, len, HASH_ENTIRE); 135 136 if (ctx) 137 put_next_job_ctx(ctx); 138 } 139 140 void 141 finish_chunk_processing(void) 142 { 143 SHA256_HASH_CTX *ctx; 144 145 while ((ctx = sha256_ctx_mgr_flush(&mb_hash_mgr)) != NULL) 146 run_fragment(ctx); 147 } 148 149 int 150 main(void) 151 { 152 int i, w, match, ret; 153 uint8_t *buffer, *p; 154 uint32_t mask, trigger, offset = 0; 155 uint32_t min_chunk, max_chunk, mean_chunk; 156 long remain; 157 struct rh_state2 state; 158 struct perf start, stop; 159 160 // Chunking parameters 161 w = 32; 162 min_chunk = 1024; 163 mean_chunk = 4 * 1024; 164 max_chunk = 32 * 1024; 165 ret = isal_rolling_hashx_mask_gen(mean_chunk, 0, &mask); 166 167 if (ret != ISAL_CRYPTO_ERR_NONE) { 168 printf(" Error generating mask"); 169 return -1; 170 } 171 172 trigger = rand() & mask; 173 174 printf("chunk and hash test w=%d, min=%d, target_ave=%d, max=%d:\n", w, min_chunk, 175 mean_chunk, max_chunk); 176 177 if (min_chunk < w || min_chunk > max_chunk) { 178 printf(" Improper parameters selected\n"); 179 return -1; 180 } 181 182 if ((buffer = malloc(MAX_BUFFER_SIZE)) == NULL) { 183 printf("cannot allocate mem\n"); 184 return -1; 185 } 186 // Initialize buffer with random data 187 srand(TEST_SEED); 188 for (i = 0; i < MAX_BUFFER_SIZE; i++) 189 buffer[i] = rand(); 190 191 // Start chunking test with multi-buffer hashing of results 192 perf_start(&start); 193 194 isal_rolling_hash2_init(&state, w); 195 setup_chunk_processing(); 196 197 p = buffer; 198 remain = MAX_BUFFER_SIZE; 199 200 while (remain > max_chunk) { 201 // Skip to min chunk 202 isal_rolling_hash2_reset(&state, p + min_chunk - w); 203 isal_rolling_hash2_run(&state, p + min_chunk, max_chunk - min_chunk, mask, trigger, 204 &offset, &match); 205 206 process_chunk(p, min_chunk + offset); 207 208 p += offset + min_chunk; 209 remain -= (offset + min_chunk); 210 } 211 212 while (remain > min_chunk) { 213 isal_rolling_hash2_reset(&state, p + min_chunk - w); 214 isal_rolling_hash2_run(&state, p + min_chunk, remain - min_chunk, mask, trigger, 215 &offset, &match); 216 217 process_chunk(p, min_chunk + offset); 218 219 p += offset + min_chunk; 220 remain -= (offset + min_chunk); 221 } 222 223 if (remain > 0) 224 process_chunk(p, remain); 225 226 finish_chunk_processing(); 227 perf_stop(&stop); 228 229 printf("chunking_with_mb_hash: "); 230 perf_print(stop, start, MAX_BUFFER_SIZE); 231 232 printf(" found %ld chunks, ave_len=%ld, filter hits=%ld\n", chunks_created, 233 MAX_BUFFER_SIZE / chunks_created, filter_hits); 234 235 return 0; 236 } 237