2 * Author......: Jens Steube <jens.steube@gmail.com>
10 #include "include/constants.h"
11 #include "include/kernel_vendor.h"
18 #include "include/kernel_functions.c"
19 #include "OpenCL/types_ocl.c"
20 #include "OpenCL/common.c"
21 #include "OpenCL/simd.c"
23 __kernel void m00110_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 combs_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
29 const u32 lid = get_local_id (0);
35 const u32 gid = get_global_id (0);
37 if (gid >= gid_max) return;
42 pws0[0] = pws[gid].i[0];
43 pws0[1] = pws[gid].i[1];
44 pws0[2] = pws[gid].i[2];
45 pws0[3] = pws[gid].i[3];
46 pws1[0] = pws[gid].i[4];
47 pws1[1] = pws[gid].i[5];
48 pws1[2] = pws[gid].i[6];
49 pws1[3] = pws[gid].i[7];
51 const u32 pw_l_len = pws[gid].pw_len;
60 salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0];
61 salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1];
62 salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2];
63 salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3];
64 salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4];
65 salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5];
66 salt_buf1[2] = salt_bufs[salt_pos].salt_buf[6];
67 salt_buf1[3] = salt_bufs[salt_pos].salt_buf[7];
69 const u32 salt_len = salt_bufs[salt_pos].salt_len;
75 for (u32 il_pos = 0; il_pos < combs_cnt; il_pos += VECT_SIZE)
77 const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos);
79 const u32x pw_len = pw_l_len + pw_r_len;
81 u32x wordr0[4] = { 0 };
82 u32x wordr1[4] = { 0 };
83 u32x wordr2[4] = { 0 };
84 u32x wordr3[4] = { 0 };
86 wordr0[0] = ix_create_combt (combs_buf, il_pos, 0);
87 wordr0[1] = ix_create_combt (combs_buf, il_pos, 1);
88 wordr0[2] = ix_create_combt (combs_buf, il_pos, 2);
89 wordr0[3] = ix_create_combt (combs_buf, il_pos, 3);
90 wordr1[0] = ix_create_combt (combs_buf, il_pos, 4);
91 wordr1[1] = ix_create_combt (combs_buf, il_pos, 5);
92 wordr1[2] = ix_create_combt (combs_buf, il_pos, 6);
93 wordr1[3] = ix_create_combt (combs_buf, il_pos, 7);
95 if (combs_mode == COMBINATOR_MODE_BASE_LEFT)
97 switch_buffer_by_offset_le (wordr0, wordr1, wordr2, wordr3, pw_l_len);
106 s0[0] = salt_buf0[0];
107 s0[1] = salt_buf0[1];
108 s0[2] = salt_buf0[2];
109 s0[3] = salt_buf0[3];
113 s1[0] = salt_buf1[0];
114 s1[1] = salt_buf1[1];
115 s1[2] = salt_buf1[2];
116 s1[3] = salt_buf1[3];
132 switch_buffer_by_offset_le_S (s0, s1, s2, s3, pw_len);
134 const u32x pw_salt_len = pw_len + salt_len;
138 w0[0] = wordl0[0] | wordr0[0] | s0[0];
139 w0[1] = wordl0[1] | wordr0[1] | s0[1];
140 w0[2] = wordl0[2] | wordr0[2] | s0[2];
141 w0[3] = wordl0[3] | wordr0[3] | s0[3];
145 w1[0] = wordl1[0] | wordr1[0] | s1[0];
146 w1[1] = wordl1[1] | wordr1[1] | s1[1];
147 w1[2] = wordl1[2] | wordr1[2] | s1[2];
148 w1[3] = wordl1[3] | wordr1[3] | s1[3];
152 w2[0] = wordl2[0] | wordr2[0] | s2[0];
153 w2[1] = wordl2[1] | wordr2[1] | s2[1];
154 w2[2] = wordl2[2] | wordr2[2] | s2[2];
155 w2[3] = wordl2[3] | wordr2[3] | s2[3];
159 w3[0] = wordl3[0] | wordr3[0] | s3[0];
160 w3[1] = wordl3[1] | wordr3[1] | s3[1];
164 append_0x80_4x4 (w0, w1, w2, w3, pw_salt_len);
170 u32x w0_t = swap32 (w0[0]);
171 u32x w1_t = swap32 (w0[1]);
172 u32x w2_t = swap32 (w0[2]);
173 u32x w3_t = swap32 (w0[3]);
174 u32x w4_t = swap32 (w1[0]);
175 u32x w5_t = swap32 (w1[1]);
176 u32x w6_t = swap32 (w1[2]);
177 u32x w7_t = swap32 (w1[3]);
178 u32x w8_t = swap32 (w2[0]);
179 u32x w9_t = swap32 (w2[1]);
180 u32x wa_t = swap32 (w2[2]);
181 u32x wb_t = swap32 (w2[3]);
182 u32x wc_t = swap32 (w3[0]);
183 u32x wd_t = swap32 (w3[1]);
185 u32x wf_t = pw_salt_len * 8;
196 SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t);
197 SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t);
198 SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t);
199 SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t);
200 SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t);
201 SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t);
202 SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t);
203 SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t);
204 SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t);
205 SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t);
206 SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t);
207 SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t);
208 SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t);
209 SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t);
210 SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t);
211 SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t);
212 w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t);
213 w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t);
214 w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t);
215 w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t);
220 w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t);
221 w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t);
222 w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t);
223 w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t);
224 w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t);
225 w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t);
226 wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t);
227 wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t);
228 wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t);
229 wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t);
230 we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t);
231 wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t);
232 w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t);
233 w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t);
234 w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t);
235 w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t);
236 w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t);
237 w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t);
238 w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t);
239 w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t);
244 w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t);
245 w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t);
246 wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t);
247 wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t);
248 wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t);
249 wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t);
250 we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t);
251 wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t);
252 w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t);
253 w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t);
254 w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t);
255 w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t);
256 w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t);
257 w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t);
258 w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t);
259 w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t);
260 w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t);
261 w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t);
262 wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t);
263 wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t);
268 wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t);
269 wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t);
270 we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t);
271 wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t);
272 w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t);
273 w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t);
274 w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t);
275 w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t);
276 w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t);
277 w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t);
278 w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t);
279 w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t);
280 w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t);
281 w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t);
282 wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t);
283 wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t);
284 wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t);
285 wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t);
286 we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t);
287 wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t);
289 COMPARE_M_SIMD (d, e, c, b);
293 __kernel void m00110_m08 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 combs_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
297 __kernel void m00110_m16 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 combs_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
301 __kernel void m00110_s04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 combs_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
307 const u32 lid = get_local_id (0);
313 const u32 gid = get_global_id (0);
315 if (gid >= gid_max) return;
320 pws0[0] = pws[gid].i[0];
321 pws0[1] = pws[gid].i[1];
322 pws0[2] = pws[gid].i[2];
323 pws0[3] = pws[gid].i[3];
324 pws1[0] = pws[gid].i[4];
325 pws1[1] = pws[gid].i[5];
326 pws1[2] = pws[gid].i[6];
327 pws1[3] = pws[gid].i[7];
329 const u32 pw_l_len = pws[gid].pw_len;
338 salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0];
339 salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1];
340 salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2];
341 salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3];
342 salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4];
343 salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5];
344 salt_buf1[2] = salt_bufs[salt_pos].salt_buf[6];
345 salt_buf1[3] = salt_bufs[salt_pos].salt_buf[7];
347 const u32 salt_len = salt_bufs[salt_pos].salt_len;
353 const u32 search[4] =
355 digests_buf[digests_offset].digest_buf[DGST_R0],
356 digests_buf[digests_offset].digest_buf[DGST_R1],
357 digests_buf[digests_offset].digest_buf[DGST_R2],
358 digests_buf[digests_offset].digest_buf[DGST_R3]
365 const u32 e_rev = rotl32_S (search[1], 2u);
371 for (u32 il_pos = 0; il_pos < combs_cnt; il_pos += VECT_SIZE)
373 const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos);
375 const u32x pw_len = pw_l_len + pw_r_len;
377 u32x wordr0[4] = { 0 };
378 u32x wordr1[4] = { 0 };
379 u32x wordr2[4] = { 0 };
380 u32x wordr3[4] = { 0 };
382 wordr0[0] = ix_create_combt (combs_buf, il_pos, 0);
383 wordr0[1] = ix_create_combt (combs_buf, il_pos, 1);
384 wordr0[2] = ix_create_combt (combs_buf, il_pos, 2);
385 wordr0[3] = ix_create_combt (combs_buf, il_pos, 3);
386 wordr1[0] = ix_create_combt (combs_buf, il_pos, 4);
387 wordr1[1] = ix_create_combt (combs_buf, il_pos, 5);
388 wordr1[2] = ix_create_combt (combs_buf, il_pos, 6);
389 wordr1[3] = ix_create_combt (combs_buf, il_pos, 7);
391 if (combs_mode == COMBINATOR_MODE_BASE_LEFT)
393 switch_buffer_by_offset_le (wordr0, wordr1, wordr2, wordr3, pw_l_len);
402 s0[0] = salt_buf0[0];
403 s0[1] = salt_buf0[1];
404 s0[2] = salt_buf0[2];
405 s0[3] = salt_buf0[3];
409 s1[0] = salt_buf1[0];
410 s1[1] = salt_buf1[1];
411 s1[2] = salt_buf1[2];
412 s1[3] = salt_buf1[3];
428 switch_buffer_by_offset_le_S (s0, s1, s2, s3, pw_len);
430 const u32x pw_salt_len = pw_len + salt_len;
434 w0[0] = wordl0[0] | wordr0[0] | s0[0];
435 w0[1] = wordl0[1] | wordr0[1] | s0[1];
436 w0[2] = wordl0[2] | wordr0[2] | s0[2];
437 w0[3] = wordl0[3] | wordr0[3] | s0[3];
441 w1[0] = wordl1[0] | wordr1[0] | s1[0];
442 w1[1] = wordl1[1] | wordr1[1] | s1[1];
443 w1[2] = wordl1[2] | wordr1[2] | s1[2];
444 w1[3] = wordl1[3] | wordr1[3] | s1[3];
448 w2[0] = wordl2[0] | wordr2[0] | s2[0];
449 w2[1] = wordl2[1] | wordr2[1] | s2[1];
450 w2[2] = wordl2[2] | wordr2[2] | s2[2];
451 w2[3] = wordl2[3] | wordr2[3] | s2[3];
455 w3[0] = wordl3[0] | wordr3[0] | s3[0];
456 w3[1] = wordl3[1] | wordr3[1] | s3[1];
460 append_0x80_4x4 (w0, w1, w2, w3, pw_salt_len);
466 u32x w0_t = swap32 (w0[0]);
467 u32x w1_t = swap32 (w0[1]);
468 u32x w2_t = swap32 (w0[2]);
469 u32x w3_t = swap32 (w0[3]);
470 u32x w4_t = swap32 (w1[0]);
471 u32x w5_t = swap32 (w1[1]);
472 u32x w6_t = swap32 (w1[2]);
473 u32x w7_t = swap32 (w1[3]);
474 u32x w8_t = swap32 (w2[0]);
475 u32x w9_t = swap32 (w2[1]);
476 u32x wa_t = swap32 (w2[2]);
477 u32x wb_t = swap32 (w2[3]);
478 u32x wc_t = swap32 (w3[0]);
479 u32x wd_t = swap32 (w3[1]);
481 u32x wf_t = pw_salt_len * 8;
492 SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t);
493 SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t);
494 SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t);
495 SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t);
496 SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t);
497 SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t);
498 SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t);
499 SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t);
500 SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t);
501 SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t);
502 SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t);
503 SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t);
504 SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t);
505 SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t);
506 SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t);
507 SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t);
508 w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t);
509 w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t);
510 w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t);
511 w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t);
516 w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t);
517 w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t);
518 w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t);
519 w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t);
520 w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t);
521 w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t);
522 wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t);
523 wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t);
524 wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t);
525 wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t);
526 we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t);
527 wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t);
528 w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t);
529 w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t);
530 w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t);
531 w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t);
532 w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t);
533 w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t);
534 w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t);
535 w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t);
540 w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t);
541 w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t);
542 wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t);
543 wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t);
544 wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t);
545 wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t);
546 we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t);
547 wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t);
548 w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t);
549 w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t);
550 w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t);
551 w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t);
552 w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t);
553 w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t);
554 w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t);
555 w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t);
556 w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t);
557 w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t);
558 wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t);
559 wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t);
564 wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t);
565 wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t);
566 we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t);
567 wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t);
568 w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t);
569 w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t);
570 w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t);
571 w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t);
572 w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t);
573 w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t);
574 w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t);
575 w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t);
576 w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t);
577 w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t);
578 wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t);
579 wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t);
581 if (MATCHES_NONE_VS (e, e_rev)) continue;
583 wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t);
584 wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t);
585 we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t);
586 wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t);
588 COMPARE_S_SIMD (d, e, c, b);
592 __kernel void m00110_s08 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 combs_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
596 __kernel void m00110_s16 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 combs_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)