xref: /isa-l_crypto/rolling_hash/chunking_with_mb_hash.c (revision 37c1320fef459b6ac01a1fae6f17aa16f54f55fb)
1 /**********************************************************************
2   Copyright(c) 2011-2017 Intel Corporation All rights reserved.
3 
4   Redistribution and use in source and binary forms, with or without
5   modification, are permitted provided that the following conditions
6   are met:
7     * Redistributions of source code must retain the above copyright
8       notice, this list of conditions and the following disclaimer.
9     * Redistributions in binary form must reproduce the above copyright
10       notice, this list of conditions and the following disclaimer in
11       the documentation and/or other materials provided with the
12       distribution.
13     * Neither the name of Intel Corporation nor the names of its
14       contributors may be used to endorse or promote products derived
15       from this software without specific prior written permission.
16 
17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29 
30 #include <stdlib.h>
31 #include <stdint.h>
32 #include <stdarg.h>
33 #include <stdio.h>
34 #include <assert.h>
35 #include "isal_crypto_api.h"
36 #include "rolling_hashx.h"
37 #include "sha256_mb.h"
38 #include "test.h"
39 
40 #define MAX_BUFFER_SIZE 128 * 1024 * 1024
41 #define HASH_POOL_SIZE  SHA256_MAX_LANES
42 
43 #ifndef TEST_SEED
44 #define TEST_SEED 0x1234
45 #endif
46 
47 #define FILTER_BITS 10
48 #define FILTER_SIZE (1 << FILTER_BITS)
49 #define FILTER_MASK (FILTER_SIZE - 1)
50 
51 #define BITS_TO_INDEX_LONG 6
52 #define MASK_TO_INDEX_LONG ((1 << BITS_TO_INDEX_LONG) - 1)
53 
54 // Globals
55 SHA256_HASH_CTX ctxpool[SHA256_MAX_LANES], *last_ctx;
56 SHA256_HASH_CTX_MGR mb_hash_mgr;
57 uint64_t filter_table[FILTER_SIZE];
58 unsigned long chunks_created = 0;
59 unsigned long filter_hits = 0;
60 
61 // Example function to run on each chunk
62 
63 void
64 run_fragment(SHA256_HASH_CTX *ctx)
65 {
66         uint64_t lookup, set_hash;
67         unsigned int lookup_hash;
68         uint32_t idx;
69 
70         chunks_created++;
71 
72         // Run a simple lookup filter on chunk using digest
73         lookup_hash = ctx->job.result_digest[0] & FILTER_MASK;
74         lookup = filter_table[lookup_hash];
75 
76         idx = ctx->job.result_digest[1];
77 
78         set_hash = 1 << (idx & MASK_TO_INDEX_LONG) |
79                    1 << ((idx >> BITS_TO_INDEX_LONG) & MASK_TO_INDEX_LONG) |
80                    1 << ((idx >> (2 * BITS_TO_INDEX_LONG)) & MASK_TO_INDEX_LONG);
81 
82         if ((lookup & set_hash) == set_hash)
83                 filter_hits++;
84         else
85                 filter_table[lookup_hash] = lookup | set_hash;
86 }
87 
88 void
89 setup_chunk_processing(void)
90 {
91         int i;
92 
93         sha256_ctx_mgr_init(&mb_hash_mgr);
94 
95         for (i = 0; i < HASH_POOL_SIZE; i++)
96                 hash_ctx_init(&ctxpool[i]);
97 
98         last_ctx = &ctxpool[0];
99 }
100 
101 SHA256_HASH_CTX *
102 get_next_job_ctx(void)
103 {
104         int i;
105         SHA256_HASH_CTX *ctx;
106 
107         if (last_ctx && hash_ctx_complete(last_ctx))
108                 return last_ctx;
109 
110         for (i = 0; i < HASH_POOL_SIZE; i++) {
111                 if (hash_ctx_complete(&ctxpool[i]))
112                         return &ctxpool[i];
113         }
114         ctx = sha256_ctx_mgr_flush(&mb_hash_mgr);
115         assert(ctx != NULL);
116         return ctx;
117 }
118 
119 void
120 put_next_job_ctx(SHA256_HASH_CTX *ctx)
121 {
122         if (ctx && hash_ctx_complete(ctx))
123                 last_ctx = ctx;
124 
125         run_fragment(ctx);
126 }
127 
128 void
129 process_chunk(uint8_t *buff, int len)
130 {
131         SHA256_HASH_CTX *ctx;
132 
133         ctx = get_next_job_ctx();
134         ctx = sha256_ctx_mgr_submit(&mb_hash_mgr, ctx, buff, len, HASH_ENTIRE);
135 
136         if (ctx)
137                 put_next_job_ctx(ctx);
138 }
139 
140 void
141 finish_chunk_processing(void)
142 {
143         SHA256_HASH_CTX *ctx;
144 
145         while ((ctx = sha256_ctx_mgr_flush(&mb_hash_mgr)) != NULL)
146                 run_fragment(ctx);
147 }
148 
149 int
150 main(void)
151 {
152         int i, w, match, ret;
153         uint8_t *buffer, *p;
154         uint32_t mask, trigger, offset = 0;
155         uint32_t min_chunk, max_chunk, mean_chunk;
156         long remain;
157         struct rh_state2 state;
158         struct perf start, stop;
159 
160         // Chunking parameters
161         w = 32;
162         min_chunk = 1024;
163         mean_chunk = 4 * 1024;
164         max_chunk = 32 * 1024;
165         ret = isal_rolling_hashx_mask_gen(mean_chunk, 0, &mask);
166 
167         if (ret != ISAL_CRYPTO_ERR_NONE) {
168                 printf(" Error generating mask");
169                 return -1;
170         }
171 
172         trigger = rand() & mask;
173 
174         printf("chunk and hash test w=%d, min=%d, target_ave=%d, max=%d:\n", w, min_chunk,
175                mean_chunk, max_chunk);
176 
177         if (min_chunk < w || min_chunk > max_chunk) {
178                 printf(" Improper parameters selected\n");
179                 return -1;
180         }
181 
182         if ((buffer = malloc(MAX_BUFFER_SIZE)) == NULL) {
183                 printf("cannot allocate mem\n");
184                 return -1;
185         }
186         // Initialize buffer with random data
187         srand(TEST_SEED);
188         for (i = 0; i < MAX_BUFFER_SIZE; i++)
189                 buffer[i] = rand();
190 
191         // Start chunking test with multi-buffer hashing of results
192         perf_start(&start);
193 
194         isal_rolling_hash2_init(&state, w);
195         setup_chunk_processing();
196 
197         p = buffer;
198         remain = MAX_BUFFER_SIZE;
199 
200         while (remain > max_chunk) {
201                 // Skip to min chunk
202                 isal_rolling_hash2_reset(&state, p + min_chunk - w);
203                 isal_rolling_hash2_run(&state, p + min_chunk, max_chunk - min_chunk, mask, trigger,
204                                        &offset, &match);
205 
206                 process_chunk(p, min_chunk + offset);
207 
208                 p += offset + min_chunk;
209                 remain -= (offset + min_chunk);
210         }
211 
212         while (remain > min_chunk) {
213                 isal_rolling_hash2_reset(&state, p + min_chunk - w);
214                 isal_rolling_hash2_run(&state, p + min_chunk, remain - min_chunk, mask, trigger,
215                                        &offset, &match);
216 
217                 process_chunk(p, min_chunk + offset);
218 
219                 p += offset + min_chunk;
220                 remain -= (offset + min_chunk);
221         }
222 
223         if (remain > 0)
224                 process_chunk(p, remain);
225 
226         finish_chunk_processing();
227         perf_stop(&stop);
228 
229         printf("chunking_with_mb_hash: ");
230         perf_print(stop, start, MAX_BUFFER_SIZE);
231 
232         printf(" found %ld chunks, ave_len=%ld, filter hits=%ld\n", chunks_created,
233                MAX_BUFFER_SIZE / chunks_created, filter_hits);
234 
235         return 0;
236 }
237