From: Jens Steube Date: Wed, 25 May 2016 21:04:26 +0000 (+0200) Subject: Move files from include/ to OpenCL/ if they are used within kernels X-Git-Tag: v3.00~149 X-Git-Url: https://www.flypig.org.uk/git/?a=commitdiff_plain;h=2899f53a1570aa14f55fc07fcaabb0476d707679;p=hashcat.git Move files from include/ to OpenCL/ if they are used within kernels Rename includes in OpenCL so that it's easier to recognize them as such --- diff --git a/OpenCL/amp_a0.cl b/OpenCL/amp_a0.cl index dbc4ad8..821c7ce 100644 --- a/OpenCL/amp_a0.cl +++ b/OpenCL/amp_a0.cl @@ -3,12 +3,12 @@ * License.....: MIT */ -#include "include/constants.h" -#include "include/kernel_vendor.h" -#include "OpenCL/types_ocl.c" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" +#include "inc_types.cl" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" +#include "inc_rp.h" +#include "inc_rp.cl" __kernel void amp (__global pw_t *pws, __global pw_t *pws_amp, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/amp_a1.cl b/OpenCL/amp_a1.cl index 553b1f6..d18e37c 100644 --- a/OpenCL/amp_a1.cl +++ b/OpenCL/amp_a1.cl @@ -3,9 +3,9 @@ * License.....: MIT */ -#include "include/constants.h" -#include "include/kernel_vendor.h" -#include "OpenCL/types_ocl.c" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" +#include "inc_types.cl" inline void switch_buffer_by_offset_le (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset) { diff --git a/OpenCL/amp_a3.cl b/OpenCL/amp_a3.cl index 370958b..f1b1b04 100644 --- a/OpenCL/amp_a3.cl +++ b/OpenCL/amp_a3.cl @@ -3,9 +3,9 @@ * License.....: MIT */ -#include "include/constants.h" -#include "include/kernel_vendor.h" -#include "OpenCL/types_ocl.c" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" +#include "inc_types.cl" __kernel void amp (__global pw_t *pws, __global pw_t *pws_amp, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/check_multi_comp4.c b/OpenCL/check_multi_comp4.c deleted file mode 100644 index 14ad55c..0000000 --- a/OpenCL/check_multi_comp4.c +++ /dev/null @@ -1,32 +0,0 @@ -u32 digest_tp[4]; - -digest_tp[0] = r0; -digest_tp[1] = r1; -digest_tp[2] = r2; -digest_tp[3] = r3; - -if (check (digest_tp, - bitmaps_buf_s1_a, - bitmaps_buf_s1_b, - bitmaps_buf_s1_c, - bitmaps_buf_s1_d, - bitmaps_buf_s2_a, - bitmaps_buf_s2_b, - bitmaps_buf_s2_c, - bitmaps_buf_s2_d, - bitmap_mask, - bitmap_shift1, - bitmap_shift2)) -{ - int digest_pos = find_hash (digest_tp, digests_cnt, &digests_buf[digests_offset]); - - if (digest_pos != -1) - { - const u32 final_hash_pos = digests_offset + digest_pos; - - if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) - { - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos); - } - } -} diff --git a/OpenCL/check_multi_comp4_bs.c b/OpenCL/check_multi_comp4_bs.c deleted file mode 100644 index 6546a97..0000000 --- a/OpenCL/check_multi_comp4_bs.c +++ /dev/null @@ -1,32 +0,0 @@ -u32 digest_tp[4]; - -digest_tp[0] = r0; -digest_tp[1] = r1; -digest_tp[2] = r2; -digest_tp[3] = r3; - -if (check (digest_tp, - bitmaps_buf_s1_a, - bitmaps_buf_s1_b, - bitmaps_buf_s1_c, - bitmaps_buf_s1_d, - bitmaps_buf_s2_a, - bitmaps_buf_s2_b, - bitmaps_buf_s2_c, - bitmaps_buf_s2_d, - bitmap_mask, - bitmap_shift1, - bitmap_shift2)) -{ - int digest_pos = find_hash (digest_tp, digests_cnt, &digests_buf[digests_offset]); - - if (digest_pos != -1) - { - const u32 final_hash_pos = digests_offset + digest_pos; - - if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) - { - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + slice); - } - } -} diff --git a/OpenCL/check_single_comp4.c b/OpenCL/check_single_comp4.c deleted file mode 100644 index 631a8bc..0000000 --- a/OpenCL/check_single_comp4.c +++ /dev/null @@ -1,12 +0,0 @@ -if ((r0 == search[0]) - && (r1 == search[1]) - && (r2 == search[2]) - && (r3 == search[3])) -{ - const u32 final_hash_pos = digests_offset + 0; - - if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) - { - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos); - } -} diff --git a/OpenCL/check_single_comp4_bs.c b/OpenCL/check_single_comp4_bs.c deleted file mode 100644 index 5c081c1..0000000 --- a/OpenCL/check_single_comp4_bs.c +++ /dev/null @@ -1,6 +0,0 @@ -const u32 final_hash_pos = digests_offset + 0; - -if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) -{ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos); -} diff --git a/OpenCL/common.c b/OpenCL/common.c deleted file mode 100644 index ca811e5..0000000 --- a/OpenCL/common.c +++ /dev/null @@ -1,9324 +0,0 @@ -/** - * Author......: Jens Steube - * License.....: MIT - */ - -/** - * pure scalar functions - */ - -inline int hash_comp (const u32 d1[4], __global u32 *d2) -{ - if (d1[3] > d2[DGST_R3]) return ( 1); - if (d1[3] < d2[DGST_R3]) return (-1); - if (d1[2] > d2[DGST_R2]) return ( 1); - if (d1[2] < d2[DGST_R2]) return (-1); - if (d1[1] > d2[DGST_R1]) return ( 1); - if (d1[1] < d2[DGST_R1]) return (-1); - if (d1[0] > d2[DGST_R0]) return ( 1); - if (d1[0] < d2[DGST_R0]) return (-1); - - return (0); -} - -inline int find_hash (const u32 digest[4], const u32 digests_cnt, __global digest_t *digests_buf) -{ - for (u32 l = 0, r = digests_cnt; r; r >>= 1) - { - const u32 m = r >> 1; - - const u32 c = l + m; - - const int cmp = hash_comp (digest, digests_buf[c].digest_buf); - - if (cmp > 0) - { - l += m + 1; - - r--; - } - - if (cmp == 0) return (c); - } - - return (-1); -} - -inline u32 check_bitmap (__global u32 *bitmap, const u32 bitmap_mask, const u32 bitmap_shift, const u32 digest) -{ - return (bitmap[(digest >> bitmap_shift) & bitmap_mask] & (1 << (digest & 0x1f))); -} - -inline u32 check (const u32 digest[2], __global u32 *bitmap_s1_a, __global u32 *bitmap_s1_b, __global u32 *bitmap_s1_c, __global u32 *bitmap_s1_d, __global u32 *bitmap_s2_a, __global u32 *bitmap_s2_b, __global u32 *bitmap_s2_c, __global u32 *bitmap_s2_d, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2) -{ - if (check_bitmap (bitmap_s1_a, bitmap_mask, bitmap_shift1, digest[0]) == 0) return (0); - if (check_bitmap (bitmap_s1_b, bitmap_mask, bitmap_shift1, digest[1]) == 0) return (0); - if (check_bitmap (bitmap_s1_c, bitmap_mask, bitmap_shift1, digest[2]) == 0) return (0); - if (check_bitmap (bitmap_s1_d, bitmap_mask, bitmap_shift1, digest[3]) == 0) return (0); - - if (check_bitmap (bitmap_s2_a, bitmap_mask, bitmap_shift2, digest[0]) == 0) return (0); - if (check_bitmap (bitmap_s2_b, bitmap_mask, bitmap_shift2, digest[1]) == 0) return (0); - if (check_bitmap (bitmap_s2_c, bitmap_mask, bitmap_shift2, digest[2]) == 0) return (0); - if (check_bitmap (bitmap_s2_d, bitmap_mask, bitmap_shift2, digest[3]) == 0) return (0); - - return (1); -} - -inline void mark_hash (__global plain_t *plains_buf, __global u32 *d_result, const int salt_pos, const int digest_pos, const int hash_pos, const u32 gid, const u32 il_pos) -{ - const u32 idx = atomic_inc (d_result); - - plains_buf[idx].salt_pos = salt_pos; - plains_buf[idx].digest_pos = digest_pos; // relative - plains_buf[idx].hash_pos = hash_pos; // absolute - plains_buf[idx].gidvid = gid; - plains_buf[idx].il_pos = il_pos; -} - -/** - * vector functions - */ - -inline void truncate_block (u32x w[4], const u32 len) -{ - switch (len) - { - case 0: w[0] &= 0; - w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 1: w[0] &= 0x000000FF; - w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 2: w[0] &= 0x0000FFFF; - w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 3: w[0] &= 0x00FFFFFF; - w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 4: w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 5: w[1] &= 0x000000FF; - w[2] &= 0; - w[3] &= 0; - break; - case 6: w[1] &= 0x0000FFFF; - w[2] &= 0; - w[3] &= 0; - break; - case 7: w[1] &= 0x00FFFFFF; - w[2] &= 0; - w[3] &= 0; - break; - case 8: w[2] &= 0; - w[3] &= 0; - break; - case 9: w[2] &= 0x000000FF; - w[3] &= 0; - break; - case 10: w[2] &= 0x0000FFFF; - w[3] &= 0; - break; - case 11: w[2] &= 0x00FFFFFF; - w[3] &= 0; - break; - case 12: w[3] &= 0; - break; - case 13: w[3] &= 0x000000FF; - break; - case 14: w[3] &= 0x0000FFFF; - break; - case 15: w[3] &= 0x00FFFFFF; - break; - } -} - -inline void make_unicode (const u32x in[4], u32x out1[4], u32x out2[4]) -{ - #ifdef IS_NV - out2[3] = __byte_perm (in[3], 0, 0x7372); - out2[2] = __byte_perm (in[3], 0, 0x7170); - out2[1] = __byte_perm (in[2], 0, 0x7372); - out2[0] = __byte_perm (in[2], 0, 0x7170); - out1[3] = __byte_perm (in[1], 0, 0x7372); - out1[2] = __byte_perm (in[1], 0, 0x7170); - out1[1] = __byte_perm (in[0], 0, 0x7372); - out1[0] = __byte_perm (in[0], 0, 0x7170); - #endif - - #if defined IS_AMD || defined IS_GENERIC - out2[3] = ((in[3] >> 8) & 0x00FF0000) | ((in[3] >> 16) & 0x000000FF); - out2[2] = ((in[3] << 8) & 0x00FF0000) | ((in[3] >> 0) & 0x000000FF); - out2[1] = ((in[2] >> 8) & 0x00FF0000) | ((in[2] >> 16) & 0x000000FF); - out2[0] = ((in[2] << 8) & 0x00FF0000) | ((in[2] >> 0) & 0x000000FF); - out1[3] = ((in[1] >> 8) & 0x00FF0000) | ((in[1] >> 16) & 0x000000FF); - out1[2] = ((in[1] << 8) & 0x00FF0000) | ((in[1] >> 0) & 0x000000FF); - out1[1] = ((in[0] >> 8) & 0x00FF0000) | ((in[0] >> 16) & 0x000000FF); - out1[0] = ((in[0] << 8) & 0x00FF0000) | ((in[0] >> 0) & 0x000000FF); - #endif -} - -inline void undo_unicode (const u32x in1[4], const u32x in2[4], u32x out[4]) -{ - #ifdef IS_NV - out[0] = __byte_perm (in1[0], in1[1], 0x6420); - out[1] = __byte_perm (in1[2], in1[3], 0x6420); - out[2] = __byte_perm (in2[0], in2[1], 0x6420); - out[3] = __byte_perm (in2[2], in2[3], 0x6420); - #endif - - #if defined IS_AMD || defined IS_GENERIC - out[0] = ((in1[0] & 0x000000ff) >> 0) | ((in1[0] & 0x00ff0000) >> 8) - | ((in1[1] & 0x000000ff) << 16) | ((in1[1] & 0x00ff0000) << 8); - out[1] = ((in1[2] & 0x000000ff) >> 0) | ((in1[2] & 0x00ff0000) >> 8) - | ((in1[3] & 0x000000ff) << 16) | ((in1[3] & 0x00ff0000) << 8); - out[2] = ((in2[0] & 0x000000ff) >> 0) | ((in2[0] & 0x00ff0000) >> 8) - | ((in2[1] & 0x000000ff) << 16) | ((in2[1] & 0x00ff0000) << 8); - out[3] = ((in2[2] & 0x000000ff) >> 0) | ((in2[2] & 0x00ff0000) >> 8) - | ((in2[3] & 0x000000ff) << 16) | ((in2[3] & 0x00ff0000) << 8); - #endif -} - -inline void append_0x01_1x4 (u32x w0[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x01; - break; - - case 1: - w0[0] = w0[0] | 0x0100; - break; - - case 2: - w0[0] = w0[0] | 0x010000; - break; - - case 3: - w0[0] = w0[0] | 0x01000000; - break; - - case 4: - w0[1] = 0x01; - break; - - case 5: - w0[1] = w0[1] | 0x0100; - break; - - case 6: - w0[1] = w0[1] | 0x010000; - break; - - case 7: - w0[1] = w0[1] | 0x01000000; - break; - - case 8: - w0[2] = 0x01; - break; - - case 9: - w0[2] = w0[2] | 0x0100; - break; - - case 10: - w0[2] = w0[2] | 0x010000; - break; - - case 11: - w0[2] = w0[2] | 0x01000000; - break; - - case 12: - w0[3] = 0x01; - break; - - case 13: - w0[3] = w0[3] | 0x0100; - break; - - case 14: - w0[3] = w0[3] | 0x010000; - break; - - case 15: - w0[3] = w0[3] | 0x01000000; - break; - } -} - -inline void append_0x01_2x4 (u32x w0[4], u32x w1[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x01; - break; - - case 1: - w0[0] = w0[0] | 0x0100; - break; - - case 2: - w0[0] = w0[0] | 0x010000; - break; - - case 3: - w0[0] = w0[0] | 0x01000000; - break; - - case 4: - w0[1] = 0x01; - break; - - case 5: - w0[1] = w0[1] | 0x0100; - break; - - case 6: - w0[1] = w0[1] | 0x010000; - break; - - case 7: - w0[1] = w0[1] | 0x01000000; - break; - - case 8: - w0[2] = 0x01; - break; - - case 9: - w0[2] = w0[2] | 0x0100; - break; - - case 10: - w0[2] = w0[2] | 0x010000; - break; - - case 11: - w0[2] = w0[2] | 0x01000000; - break; - - case 12: - w0[3] = 0x01; - break; - - case 13: - w0[3] = w0[3] | 0x0100; - break; - - case 14: - w0[3] = w0[3] | 0x010000; - break; - - case 15: - w0[3] = w0[3] | 0x01000000; - break; - - case 16: - w1[0] = 0x01; - break; - - case 17: - w1[0] = w1[0] | 0x0100; - break; - - case 18: - w1[0] = w1[0] | 0x010000; - break; - - case 19: - w1[0] = w1[0] | 0x01000000; - break; - - case 20: - w1[1] = 0x01; - break; - - case 21: - w1[1] = w1[1] | 0x0100; - break; - - case 22: - w1[1] = w1[1] | 0x010000; - break; - - case 23: - w1[1] = w1[1] | 0x01000000; - break; - - case 24: - w1[2] = 0x01; - break; - - case 25: - w1[2] = w1[2] | 0x0100; - break; - - case 26: - w1[2] = w1[2] | 0x010000; - break; - - case 27: - w1[2] = w1[2] | 0x01000000; - break; - - case 28: - w1[3] = 0x01; - break; - - case 29: - w1[3] = w1[3] | 0x0100; - break; - - case 30: - w1[3] = w1[3] | 0x010000; - break; - - case 31: - w1[3] = w1[3] | 0x01000000; - break; - } -} - -inline void append_0x01_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x01; - break; - - case 1: - w0[0] = w0[0] | 0x0100; - break; - - case 2: - w0[0] = w0[0] | 0x010000; - break; - - case 3: - w0[0] = w0[0] | 0x01000000; - break; - - case 4: - w0[1] = 0x01; - break; - - case 5: - w0[1] = w0[1] | 0x0100; - break; - - case 6: - w0[1] = w0[1] | 0x010000; - break; - - case 7: - w0[1] = w0[1] | 0x01000000; - break; - - case 8: - w0[2] = 0x01; - break; - - case 9: - w0[2] = w0[2] | 0x0100; - break; - - case 10: - w0[2] = w0[2] | 0x010000; - break; - - case 11: - w0[2] = w0[2] | 0x01000000; - break; - - case 12: - w0[3] = 0x01; - break; - - case 13: - w0[3] = w0[3] | 0x0100; - break; - - case 14: - w0[3] = w0[3] | 0x010000; - break; - - case 15: - w0[3] = w0[3] | 0x01000000; - break; - - case 16: - w1[0] = 0x01; - break; - - case 17: - w1[0] = w1[0] | 0x0100; - break; - - case 18: - w1[0] = w1[0] | 0x010000; - break; - - case 19: - w1[0] = w1[0] | 0x01000000; - break; - - case 20: - w1[1] = 0x01; - break; - - case 21: - w1[1] = w1[1] | 0x0100; - break; - - case 22: - w1[1] = w1[1] | 0x010000; - break; - - case 23: - w1[1] = w1[1] | 0x01000000; - break; - - case 24: - w1[2] = 0x01; - break; - - case 25: - w1[2] = w1[2] | 0x0100; - break; - - case 26: - w1[2] = w1[2] | 0x010000; - break; - - case 27: - w1[2] = w1[2] | 0x01000000; - break; - - case 28: - w1[3] = 0x01; - break; - - case 29: - w1[3] = w1[3] | 0x0100; - break; - - case 30: - w1[3] = w1[3] | 0x010000; - break; - - case 31: - w1[3] = w1[3] | 0x01000000; - break; - - case 32: - w2[0] = 0x01; - break; - - case 33: - w2[0] = w2[0] | 0x0100; - break; - - case 34: - w2[0] = w2[0] | 0x010000; - break; - - case 35: - w2[0] = w2[0] | 0x01000000; - break; - - case 36: - w2[1] = 0x01; - break; - - case 37: - w2[1] = w2[1] | 0x0100; - break; - - case 38: - w2[1] = w2[1] | 0x010000; - break; - - case 39: - w2[1] = w2[1] | 0x01000000; - break; - - case 40: - w2[2] = 0x01; - break; - - case 41: - w2[2] = w2[2] | 0x0100; - break; - - case 42: - w2[2] = w2[2] | 0x010000; - break; - - case 43: - w2[2] = w2[2] | 0x01000000; - break; - - case 44: - w2[3] = 0x01; - break; - - case 45: - w2[3] = w2[3] | 0x0100; - break; - - case 46: - w2[3] = w2[3] | 0x010000; - break; - - case 47: - w2[3] = w2[3] | 0x01000000; - break; - } -} - -inline void append_0x01_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x01; - break; - - case 1: - w0[0] = w0[0] | 0x0100; - break; - - case 2: - w0[0] = w0[0] | 0x010000; - break; - - case 3: - w0[0] = w0[0] | 0x01000000; - break; - - case 4: - w0[1] = 0x01; - break; - - case 5: - w0[1] = w0[1] | 0x0100; - break; - - case 6: - w0[1] = w0[1] | 0x010000; - break; - - case 7: - w0[1] = w0[1] | 0x01000000; - break; - - case 8: - w0[2] = 0x01; - break; - - case 9: - w0[2] = w0[2] | 0x0100; - break; - - case 10: - w0[2] = w0[2] | 0x010000; - break; - - case 11: - w0[2] = w0[2] | 0x01000000; - break; - - case 12: - w0[3] = 0x01; - break; - - case 13: - w0[3] = w0[3] | 0x0100; - break; - - case 14: - w0[3] = w0[3] | 0x010000; - break; - - case 15: - w0[3] = w0[3] | 0x01000000; - break; - - case 16: - w1[0] = 0x01; - break; - - case 17: - w1[0] = w1[0] | 0x0100; - break; - - case 18: - w1[0] = w1[0] | 0x010000; - break; - - case 19: - w1[0] = w1[0] | 0x01000000; - break; - - case 20: - w1[1] = 0x01; - break; - - case 21: - w1[1] = w1[1] | 0x0100; - break; - - case 22: - w1[1] = w1[1] | 0x010000; - break; - - case 23: - w1[1] = w1[1] | 0x01000000; - break; - - case 24: - w1[2] = 0x01; - break; - - case 25: - w1[2] = w1[2] | 0x0100; - break; - - case 26: - w1[2] = w1[2] | 0x010000; - break; - - case 27: - w1[2] = w1[2] | 0x01000000; - break; - - case 28: - w1[3] = 0x01; - break; - - case 29: - w1[3] = w1[3] | 0x0100; - break; - - case 30: - w1[3] = w1[3] | 0x010000; - break; - - case 31: - w1[3] = w1[3] | 0x01000000; - break; - - case 32: - w2[0] = 0x01; - break; - - case 33: - w2[0] = w2[0] | 0x0100; - break; - - case 34: - w2[0] = w2[0] | 0x010000; - break; - - case 35: - w2[0] = w2[0] | 0x01000000; - break; - - case 36: - w2[1] = 0x01; - break; - - case 37: - w2[1] = w2[1] | 0x0100; - break; - - case 38: - w2[1] = w2[1] | 0x010000; - break; - - case 39: - w2[1] = w2[1] | 0x01000000; - break; - - case 40: - w2[2] = 0x01; - break; - - case 41: - w2[2] = w2[2] | 0x0100; - break; - - case 42: - w2[2] = w2[2] | 0x010000; - break; - - case 43: - w2[2] = w2[2] | 0x01000000; - break; - - case 44: - w2[3] = 0x01; - break; - - case 45: - w2[3] = w2[3] | 0x0100; - break; - - case 46: - w2[3] = w2[3] | 0x010000; - break; - - case 47: - w2[3] = w2[3] | 0x01000000; - break; - - case 48: - w3[0] = 0x01; - break; - - case 49: - w3[0] = w3[0] | 0x0100; - break; - - case 50: - w3[0] = w3[0] | 0x010000; - break; - - case 51: - w3[0] = w3[0] | 0x01000000; - break; - - case 52: - w3[1] = 0x01; - break; - - case 53: - w3[1] = w3[1] | 0x0100; - break; - - case 54: - w3[1] = w3[1] | 0x010000; - break; - - case 55: - w3[1] = w3[1] | 0x01000000; - break; - - case 56: - w3[2] = 0x01; - break; - - case 57: - w3[2] = w3[2] | 0x0100; - break; - - case 58: - w3[2] = w3[2] | 0x010000; - break; - - case 59: - w3[2] = w3[2] | 0x01000000; - break; - - case 60: - w3[3] = 0x01; - break; - - case 61: - w3[3] = w3[3] | 0x0100; - break; - - case 62: - w3[3] = w3[3] | 0x010000; - break; - - case 63: - w3[3] = w3[3] | 0x01000000; - break; - } -} - -inline void append_0x01_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x01; - break; - - case 1: - w0[0] = w0[0] | 0x0100; - break; - - case 2: - w0[0] = w0[0] | 0x010000; - break; - - case 3: - w0[0] = w0[0] | 0x01000000; - break; - - case 4: - w0[1] = 0x01; - break; - - case 5: - w0[1] = w0[1] | 0x0100; - break; - - case 6: - w0[1] = w0[1] | 0x010000; - break; - - case 7: - w0[1] = w0[1] | 0x01000000; - break; - - case 8: - w0[2] = 0x01; - break; - - case 9: - w0[2] = w0[2] | 0x0100; - break; - - case 10: - w0[2] = w0[2] | 0x010000; - break; - - case 11: - w0[2] = w0[2] | 0x01000000; - break; - - case 12: - w0[3] = 0x01; - break; - - case 13: - w0[3] = w0[3] | 0x0100; - break; - - case 14: - w0[3] = w0[3] | 0x010000; - break; - - case 15: - w0[3] = w0[3] | 0x01000000; - break; - - case 16: - w1[0] = 0x01; - break; - - case 17: - w1[0] = w1[0] | 0x0100; - break; - - case 18: - w1[0] = w1[0] | 0x010000; - break; - - case 19: - w1[0] = w1[0] | 0x01000000; - break; - - case 20: - w1[1] = 0x01; - break; - - case 21: - w1[1] = w1[1] | 0x0100; - break; - - case 22: - w1[1] = w1[1] | 0x010000; - break; - - case 23: - w1[1] = w1[1] | 0x01000000; - break; - - case 24: - w1[2] = 0x01; - break; - - case 25: - w1[2] = w1[2] | 0x0100; - break; - - case 26: - w1[2] = w1[2] | 0x010000; - break; - - case 27: - w1[2] = w1[2] | 0x01000000; - break; - - case 28: - w1[3] = 0x01; - break; - - case 29: - w1[3] = w1[3] | 0x0100; - break; - - case 30: - w1[3] = w1[3] | 0x010000; - break; - - case 31: - w1[3] = w1[3] | 0x01000000; - break; - - case 32: - w2[0] = 0x01; - break; - - case 33: - w2[0] = w2[0] | 0x0100; - break; - - case 34: - w2[0] = w2[0] | 0x010000; - break; - - case 35: - w2[0] = w2[0] | 0x01000000; - break; - - case 36: - w2[1] = 0x01; - break; - - case 37: - w2[1] = w2[1] | 0x0100; - break; - - case 38: - w2[1] = w2[1] | 0x010000; - break; - - case 39: - w2[1] = w2[1] | 0x01000000; - break; - - case 40: - w2[2] = 0x01; - break; - - case 41: - w2[2] = w2[2] | 0x0100; - break; - - case 42: - w2[2] = w2[2] | 0x010000; - break; - - case 43: - w2[2] = w2[2] | 0x01000000; - break; - - case 44: - w2[3] = 0x01; - break; - - case 45: - w2[3] = w2[3] | 0x0100; - break; - - case 46: - w2[3] = w2[3] | 0x010000; - break; - - case 47: - w2[3] = w2[3] | 0x01000000; - break; - - case 48: - w3[0] = 0x01; - break; - - case 49: - w3[0] = w3[0] | 0x0100; - break; - - case 50: - w3[0] = w3[0] | 0x010000; - break; - - case 51: - w3[0] = w3[0] | 0x01000000; - break; - - case 52: - w3[1] = 0x01; - break; - - case 53: - w3[1] = w3[1] | 0x0100; - break; - - case 54: - w3[1] = w3[1] | 0x010000; - break; - - case 55: - w3[1] = w3[1] | 0x01000000; - break; - - case 56: - w3[2] = 0x01; - break; - - case 57: - w3[2] = w3[2] | 0x0100; - break; - - case 58: - w3[2] = w3[2] | 0x010000; - break; - - case 59: - w3[2] = w3[2] | 0x01000000; - break; - - case 60: - w3[3] = 0x01; - break; - - case 61: - w3[3] = w3[3] | 0x0100; - break; - - case 62: - w3[3] = w3[3] | 0x010000; - break; - - case 63: - w3[3] = w3[3] | 0x01000000; - break; - - case 64: - w4[0] = 0x01; - break; - - case 65: - w4[0] = w4[0] | 0x0100; - break; - - case 66: - w4[0] = w4[0] | 0x010000; - break; - - case 67: - w4[0] = w4[0] | 0x01000000; - break; - - case 68: - w4[1] = 0x01; - break; - - case 69: - w4[1] = w4[1] | 0x0100; - break; - - case 70: - w4[1] = w4[1] | 0x010000; - break; - - case 71: - w4[1] = w4[1] | 0x01000000; - break; - - case 72: - w4[2] = 0x01; - break; - - case 73: - w4[2] = w4[2] | 0x0100; - break; - - case 74: - w4[2] = w4[2] | 0x010000; - break; - - case 75: - w4[2] = w4[2] | 0x01000000; - break; - - case 76: - w4[3] = 0x01; - break; - - case 77: - w4[3] = w4[3] | 0x0100; - break; - - case 78: - w4[3] = w4[3] | 0x010000; - break; - - case 79: - w4[3] = w4[3] | 0x01000000; - break; - - case 80: - w5[0] = 0x01; - break; - - case 81: - w5[0] = w5[0] | 0x0100; - break; - - case 82: - w5[0] = w5[0] | 0x010000; - break; - - case 83: - w5[0] = w5[0] | 0x01000000; - break; - - case 84: - w5[1] = 0x01; - break; - - case 85: - w5[1] = w5[1] | 0x0100; - break; - - case 86: - w5[1] = w5[1] | 0x010000; - break; - - case 87: - w5[1] = w5[1] | 0x01000000; - break; - - case 88: - w5[2] = 0x01; - break; - - case 89: - w5[2] = w5[2] | 0x0100; - break; - - case 90: - w5[2] = w5[2] | 0x010000; - break; - - case 91: - w5[2] = w5[2] | 0x01000000; - break; - - case 92: - w5[3] = 0x01; - break; - - case 93: - w5[3] = w5[3] | 0x0100; - break; - - case 94: - w5[3] = w5[3] | 0x010000; - break; - - case 95: - w5[3] = w5[3] | 0x01000000; - break; - - case 96: - w6[0] = 0x01; - break; - - case 97: - w6[0] = w6[0] | 0x0100; - break; - - case 98: - w6[0] = w6[0] | 0x010000; - break; - - case 99: - w6[0] = w6[0] | 0x01000000; - break; - - case 100: - w6[1] = 0x01; - break; - - case 101: - w6[1] = w6[1] | 0x0100; - break; - - case 102: - w6[1] = w6[1] | 0x010000; - break; - - case 103: - w6[1] = w6[1] | 0x01000000; - break; - - case 104: - w6[2] = 0x01; - break; - - case 105: - w6[2] = w6[2] | 0x0100; - break; - - case 106: - w6[2] = w6[2] | 0x010000; - break; - - case 107: - w6[2] = w6[2] | 0x01000000; - break; - - case 108: - w6[3] = 0x01; - break; - - case 109: - w6[3] = w6[3] | 0x0100; - break; - - case 110: - w6[3] = w6[3] | 0x010000; - break; - - case 111: - w6[3] = w6[3] | 0x01000000; - break; - - case 112: - w7[0] = 0x01; - break; - - case 113: - w7[0] = w7[0] | 0x0100; - break; - - case 114: - w7[0] = w7[0] | 0x010000; - break; - - case 115: - w7[0] = w7[0] | 0x01000000; - break; - - case 116: - w7[1] = 0x01; - break; - - case 117: - w7[1] = w7[1] | 0x0100; - break; - - case 118: - w7[1] = w7[1] | 0x010000; - break; - - case 119: - w7[1] = w7[1] | 0x01000000; - break; - - case 120: - w7[2] = 0x01; - break; - - case 121: - w7[2] = w7[2] | 0x0100; - break; - - case 122: - w7[2] = w7[2] | 0x010000; - break; - - case 123: - w7[2] = w7[2] | 0x01000000; - break; - - case 124: - w7[3] = 0x01; - break; - - case 125: - w7[3] = w7[3] | 0x0100; - break; - - case 126: - w7[3] = w7[3] | 0x010000; - break; - - case 127: - w7[3] = w7[3] | 0x01000000; - break; - } -} - -inline void append_0x02_1x4 (u32x w0[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x02; - break; - - case 1: - w0[0] = w0[0] | 0x0200; - break; - - case 2: - w0[0] = w0[0] | 0x020000; - break; - - case 3: - w0[0] = w0[0] | 0x02000000; - break; - - case 4: - w0[1] = 0x02; - break; - - case 5: - w0[1] = w0[1] | 0x0200; - break; - - case 6: - w0[1] = w0[1] | 0x020000; - break; - - case 7: - w0[1] = w0[1] | 0x02000000; - break; - - case 8: - w0[2] = 0x02; - break; - - case 9: - w0[2] = w0[2] | 0x0200; - break; - - case 10: - w0[2] = w0[2] | 0x020000; - break; - - case 11: - w0[2] = w0[2] | 0x02000000; - break; - - case 12: - w0[3] = 0x02; - break; - - case 13: - w0[3] = w0[3] | 0x0200; - break; - - case 14: - w0[3] = w0[3] | 0x020000; - break; - - case 15: - w0[3] = w0[3] | 0x02000000; - break; - } -} - -inline void append_0x02_2x4 (u32x w0[4], u32x w1[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x02; - break; - - case 1: - w0[0] = w0[0] | 0x0200; - break; - - case 2: - w0[0] = w0[0] | 0x020000; - break; - - case 3: - w0[0] = w0[0] | 0x02000000; - break; - - case 4: - w0[1] = 0x02; - break; - - case 5: - w0[1] = w0[1] | 0x0200; - break; - - case 6: - w0[1] = w0[1] | 0x020000; - break; - - case 7: - w0[1] = w0[1] | 0x02000000; - break; - - case 8: - w0[2] = 0x02; - break; - - case 9: - w0[2] = w0[2] | 0x0200; - break; - - case 10: - w0[2] = w0[2] | 0x020000; - break; - - case 11: - w0[2] = w0[2] | 0x02000000; - break; - - case 12: - w0[3] = 0x02; - break; - - case 13: - w0[3] = w0[3] | 0x0200; - break; - - case 14: - w0[3] = w0[3] | 0x020000; - break; - - case 15: - w0[3] = w0[3] | 0x02000000; - break; - - case 16: - w1[0] = 0x02; - break; - - case 17: - w1[0] = w1[0] | 0x0200; - break; - - case 18: - w1[0] = w1[0] | 0x020000; - break; - - case 19: - w1[0] = w1[0] | 0x02000000; - break; - - case 20: - w1[1] = 0x02; - break; - - case 21: - w1[1] = w1[1] | 0x0200; - break; - - case 22: - w1[1] = w1[1] | 0x020000; - break; - - case 23: - w1[1] = w1[1] | 0x02000000; - break; - - case 24: - w1[2] = 0x02; - break; - - case 25: - w1[2] = w1[2] | 0x0200; - break; - - case 26: - w1[2] = w1[2] | 0x020000; - break; - - case 27: - w1[2] = w1[2] | 0x02000000; - break; - - case 28: - w1[3] = 0x02; - break; - - case 29: - w1[3] = w1[3] | 0x0200; - break; - - case 30: - w1[3] = w1[3] | 0x020000; - break; - - case 31: - w1[3] = w1[3] | 0x02000000; - break; - } -} - -inline void append_0x02_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x02; - break; - - case 1: - w0[0] = w0[0] | 0x0200; - break; - - case 2: - w0[0] = w0[0] | 0x020000; - break; - - case 3: - w0[0] = w0[0] | 0x02000000; - break; - - case 4: - w0[1] = 0x02; - break; - - case 5: - w0[1] = w0[1] | 0x0200; - break; - - case 6: - w0[1] = w0[1] | 0x020000; - break; - - case 7: - w0[1] = w0[1] | 0x02000000; - break; - - case 8: - w0[2] = 0x02; - break; - - case 9: - w0[2] = w0[2] | 0x0200; - break; - - case 10: - w0[2] = w0[2] | 0x020000; - break; - - case 11: - w0[2] = w0[2] | 0x02000000; - break; - - case 12: - w0[3] = 0x02; - break; - - case 13: - w0[3] = w0[3] | 0x0200; - break; - - case 14: - w0[3] = w0[3] | 0x020000; - break; - - case 15: - w0[3] = w0[3] | 0x02000000; - break; - - case 16: - w1[0] = 0x02; - break; - - case 17: - w1[0] = w1[0] | 0x0200; - break; - - case 18: - w1[0] = w1[0] | 0x020000; - break; - - case 19: - w1[0] = w1[0] | 0x02000000; - break; - - case 20: - w1[1] = 0x02; - break; - - case 21: - w1[1] = w1[1] | 0x0200; - break; - - case 22: - w1[1] = w1[1] | 0x020000; - break; - - case 23: - w1[1] = w1[1] | 0x02000000; - break; - - case 24: - w1[2] = 0x02; - break; - - case 25: - w1[2] = w1[2] | 0x0200; - break; - - case 26: - w1[2] = w1[2] | 0x020000; - break; - - case 27: - w1[2] = w1[2] | 0x02000000; - break; - - case 28: - w1[3] = 0x02; - break; - - case 29: - w1[3] = w1[3] | 0x0200; - break; - - case 30: - w1[3] = w1[3] | 0x020000; - break; - - case 31: - w1[3] = w1[3] | 0x02000000; - break; - - case 32: - w2[0] = 0x02; - break; - - case 33: - w2[0] = w2[0] | 0x0200; - break; - - case 34: - w2[0] = w2[0] | 0x020000; - break; - - case 35: - w2[0] = w2[0] | 0x02000000; - break; - - case 36: - w2[1] = 0x02; - break; - - case 37: - w2[1] = w2[1] | 0x0200; - break; - - case 38: - w2[1] = w2[1] | 0x020000; - break; - - case 39: - w2[1] = w2[1] | 0x02000000; - break; - - case 40: - w2[2] = 0x02; - break; - - case 41: - w2[2] = w2[2] | 0x0200; - break; - - case 42: - w2[2] = w2[2] | 0x020000; - break; - - case 43: - w2[2] = w2[2] | 0x02000000; - break; - - case 44: - w2[3] = 0x02; - break; - - case 45: - w2[3] = w2[3] | 0x0200; - break; - - case 46: - w2[3] = w2[3] | 0x020000; - break; - - case 47: - w2[3] = w2[3] | 0x02000000; - break; - } -} - -inline void append_0x02_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x02; - break; - - case 1: - w0[0] = w0[0] | 0x0200; - break; - - case 2: - w0[0] = w0[0] | 0x020000; - break; - - case 3: - w0[0] = w0[0] | 0x02000000; - break; - - case 4: - w0[1] = 0x02; - break; - - case 5: - w0[1] = w0[1] | 0x0200; - break; - - case 6: - w0[1] = w0[1] | 0x020000; - break; - - case 7: - w0[1] = w0[1] | 0x02000000; - break; - - case 8: - w0[2] = 0x02; - break; - - case 9: - w0[2] = w0[2] | 0x0200; - break; - - case 10: - w0[2] = w0[2] | 0x020000; - break; - - case 11: - w0[2] = w0[2] | 0x02000000; - break; - - case 12: - w0[3] = 0x02; - break; - - case 13: - w0[3] = w0[3] | 0x0200; - break; - - case 14: - w0[3] = w0[3] | 0x020000; - break; - - case 15: - w0[3] = w0[3] | 0x02000000; - break; - - case 16: - w1[0] = 0x02; - break; - - case 17: - w1[0] = w1[0] | 0x0200; - break; - - case 18: - w1[0] = w1[0] | 0x020000; - break; - - case 19: - w1[0] = w1[0] | 0x02000000; - break; - - case 20: - w1[1] = 0x02; - break; - - case 21: - w1[1] = w1[1] | 0x0200; - break; - - case 22: - w1[1] = w1[1] | 0x020000; - break; - - case 23: - w1[1] = w1[1] | 0x02000000; - break; - - case 24: - w1[2] = 0x02; - break; - - case 25: - w1[2] = w1[2] | 0x0200; - break; - - case 26: - w1[2] = w1[2] | 0x020000; - break; - - case 27: - w1[2] = w1[2] | 0x02000000; - break; - - case 28: - w1[3] = 0x02; - break; - - case 29: - w1[3] = w1[3] | 0x0200; - break; - - case 30: - w1[3] = w1[3] | 0x020000; - break; - - case 31: - w1[3] = w1[3] | 0x02000000; - break; - - case 32: - w2[0] = 0x02; - break; - - case 33: - w2[0] = w2[0] | 0x0200; - break; - - case 34: - w2[0] = w2[0] | 0x020000; - break; - - case 35: - w2[0] = w2[0] | 0x02000000; - break; - - case 36: - w2[1] = 0x02; - break; - - case 37: - w2[1] = w2[1] | 0x0200; - break; - - case 38: - w2[1] = w2[1] | 0x020000; - break; - - case 39: - w2[1] = w2[1] | 0x02000000; - break; - - case 40: - w2[2] = 0x02; - break; - - case 41: - w2[2] = w2[2] | 0x0200; - break; - - case 42: - w2[2] = w2[2] | 0x020000; - break; - - case 43: - w2[2] = w2[2] | 0x02000000; - break; - - case 44: - w2[3] = 0x02; - break; - - case 45: - w2[3] = w2[3] | 0x0200; - break; - - case 46: - w2[3] = w2[3] | 0x020000; - break; - - case 47: - w2[3] = w2[3] | 0x02000000; - break; - - case 48: - w3[0] = 0x02; - break; - - case 49: - w3[0] = w3[0] | 0x0200; - break; - - case 50: - w3[0] = w3[0] | 0x020000; - break; - - case 51: - w3[0] = w3[0] | 0x02000000; - break; - - case 52: - w3[1] = 0x02; - break; - - case 53: - w3[1] = w3[1] | 0x0200; - break; - - case 54: - w3[1] = w3[1] | 0x020000; - break; - - case 55: - w3[1] = w3[1] | 0x02000000; - break; - - case 56: - w3[2] = 0x02; - break; - - case 57: - w3[2] = w3[2] | 0x0200; - break; - - case 58: - w3[2] = w3[2] | 0x020000; - break; - - case 59: - w3[2] = w3[2] | 0x02000000; - break; - - case 60: - w3[3] = 0x02; - break; - - case 61: - w3[3] = w3[3] | 0x0200; - break; - - case 62: - w3[3] = w3[3] | 0x020000; - break; - - case 63: - w3[3] = w3[3] | 0x02000000; - break; - } -} - -inline void append_0x02_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x02; - break; - - case 1: - w0[0] = w0[0] | 0x0200; - break; - - case 2: - w0[0] = w0[0] | 0x020000; - break; - - case 3: - w0[0] = w0[0] | 0x02000000; - break; - - case 4: - w0[1] = 0x02; - break; - - case 5: - w0[1] = w0[1] | 0x0200; - break; - - case 6: - w0[1] = w0[1] | 0x020000; - break; - - case 7: - w0[1] = w0[1] | 0x02000000; - break; - - case 8: - w0[2] = 0x02; - break; - - case 9: - w0[2] = w0[2] | 0x0200; - break; - - case 10: - w0[2] = w0[2] | 0x020000; - break; - - case 11: - w0[2] = w0[2] | 0x02000000; - break; - - case 12: - w0[3] = 0x02; - break; - - case 13: - w0[3] = w0[3] | 0x0200; - break; - - case 14: - w0[3] = w0[3] | 0x020000; - break; - - case 15: - w0[3] = w0[3] | 0x02000000; - break; - - case 16: - w1[0] = 0x02; - break; - - case 17: - w1[0] = w1[0] | 0x0200; - break; - - case 18: - w1[0] = w1[0] | 0x020000; - break; - - case 19: - w1[0] = w1[0] | 0x02000000; - break; - - case 20: - w1[1] = 0x02; - break; - - case 21: - w1[1] = w1[1] | 0x0200; - break; - - case 22: - w1[1] = w1[1] | 0x020000; - break; - - case 23: - w1[1] = w1[1] | 0x02000000; - break; - - case 24: - w1[2] = 0x02; - break; - - case 25: - w1[2] = w1[2] | 0x0200; - break; - - case 26: - w1[2] = w1[2] | 0x020000; - break; - - case 27: - w1[2] = w1[2] | 0x02000000; - break; - - case 28: - w1[3] = 0x02; - break; - - case 29: - w1[3] = w1[3] | 0x0200; - break; - - case 30: - w1[3] = w1[3] | 0x020000; - break; - - case 31: - w1[3] = w1[3] | 0x02000000; - break; - - case 32: - w2[0] = 0x02; - break; - - case 33: - w2[0] = w2[0] | 0x0200; - break; - - case 34: - w2[0] = w2[0] | 0x020000; - break; - - case 35: - w2[0] = w2[0] | 0x02000000; - break; - - case 36: - w2[1] = 0x02; - break; - - case 37: - w2[1] = w2[1] | 0x0200; - break; - - case 38: - w2[1] = w2[1] | 0x020000; - break; - - case 39: - w2[1] = w2[1] | 0x02000000; - break; - - case 40: - w2[2] = 0x02; - break; - - case 41: - w2[2] = w2[2] | 0x0200; - break; - - case 42: - w2[2] = w2[2] | 0x020000; - break; - - case 43: - w2[2] = w2[2] | 0x02000000; - break; - - case 44: - w2[3] = 0x02; - break; - - case 45: - w2[3] = w2[3] | 0x0200; - break; - - case 46: - w2[3] = w2[3] | 0x020000; - break; - - case 47: - w2[3] = w2[3] | 0x02000000; - break; - - case 48: - w3[0] = 0x02; - break; - - case 49: - w3[0] = w3[0] | 0x0200; - break; - - case 50: - w3[0] = w3[0] | 0x020000; - break; - - case 51: - w3[0] = w3[0] | 0x02000000; - break; - - case 52: - w3[1] = 0x02; - break; - - case 53: - w3[1] = w3[1] | 0x0200; - break; - - case 54: - w3[1] = w3[1] | 0x020000; - break; - - case 55: - w3[1] = w3[1] | 0x02000000; - break; - - case 56: - w3[2] = 0x02; - break; - - case 57: - w3[2] = w3[2] | 0x0200; - break; - - case 58: - w3[2] = w3[2] | 0x020000; - break; - - case 59: - w3[2] = w3[2] | 0x02000000; - break; - - case 60: - w3[3] = 0x02; - break; - - case 61: - w3[3] = w3[3] | 0x0200; - break; - - case 62: - w3[3] = w3[3] | 0x020000; - break; - - case 63: - w3[3] = w3[3] | 0x02000000; - break; - - case 64: - w4[0] = 0x02; - break; - - case 65: - w4[0] = w4[0] | 0x0200; - break; - - case 66: - w4[0] = w4[0] | 0x020000; - break; - - case 67: - w4[0] = w4[0] | 0x02000000; - break; - - case 68: - w4[1] = 0x02; - break; - - case 69: - w4[1] = w4[1] | 0x0200; - break; - - case 70: - w4[1] = w4[1] | 0x020000; - break; - - case 71: - w4[1] = w4[1] | 0x02000000; - break; - - case 72: - w4[2] = 0x02; - break; - - case 73: - w4[2] = w4[2] | 0x0200; - break; - - case 74: - w4[2] = w4[2] | 0x020000; - break; - - case 75: - w4[2] = w4[2] | 0x02000000; - break; - - case 76: - w4[3] = 0x02; - break; - - case 77: - w4[3] = w4[3] | 0x0200; - break; - - case 78: - w4[3] = w4[3] | 0x020000; - break; - - case 79: - w4[3] = w4[3] | 0x02000000; - break; - - case 80: - w5[0] = 0x02; - break; - - case 81: - w5[0] = w5[0] | 0x0200; - break; - - case 82: - w5[0] = w5[0] | 0x020000; - break; - - case 83: - w5[0] = w5[0] | 0x02000000; - break; - - case 84: - w5[1] = 0x02; - break; - - case 85: - w5[1] = w5[1] | 0x0200; - break; - - case 86: - w5[1] = w5[1] | 0x020000; - break; - - case 87: - w5[1] = w5[1] | 0x02000000; - break; - - case 88: - w5[2] = 0x02; - break; - - case 89: - w5[2] = w5[2] | 0x0200; - break; - - case 90: - w5[2] = w5[2] | 0x020000; - break; - - case 91: - w5[2] = w5[2] | 0x02000000; - break; - - case 92: - w5[3] = 0x02; - break; - - case 93: - w5[3] = w5[3] | 0x0200; - break; - - case 94: - w5[3] = w5[3] | 0x020000; - break; - - case 95: - w5[3] = w5[3] | 0x02000000; - break; - - case 96: - w6[0] = 0x02; - break; - - case 97: - w6[0] = w6[0] | 0x0200; - break; - - case 98: - w6[0] = w6[0] | 0x020000; - break; - - case 99: - w6[0] = w6[0] | 0x02000000; - break; - - case 100: - w6[1] = 0x02; - break; - - case 101: - w6[1] = w6[1] | 0x0200; - break; - - case 102: - w6[1] = w6[1] | 0x020000; - break; - - case 103: - w6[1] = w6[1] | 0x02000000; - break; - - case 104: - w6[2] = 0x02; - break; - - case 105: - w6[2] = w6[2] | 0x0200; - break; - - case 106: - w6[2] = w6[2] | 0x020000; - break; - - case 107: - w6[2] = w6[2] | 0x02000000; - break; - - case 108: - w6[3] = 0x02; - break; - - case 109: - w6[3] = w6[3] | 0x0200; - break; - - case 110: - w6[3] = w6[3] | 0x020000; - break; - - case 111: - w6[3] = w6[3] | 0x02000000; - break; - - case 112: - w7[0] = 0x02; - break; - - case 113: - w7[0] = w7[0] | 0x0200; - break; - - case 114: - w7[0] = w7[0] | 0x020000; - break; - - case 115: - w7[0] = w7[0] | 0x02000000; - break; - - case 116: - w7[1] = 0x02; - break; - - case 117: - w7[1] = w7[1] | 0x0200; - break; - - case 118: - w7[1] = w7[1] | 0x020000; - break; - - case 119: - w7[1] = w7[1] | 0x02000000; - break; - - case 120: - w7[2] = 0x02; - break; - - case 121: - w7[2] = w7[2] | 0x0200; - break; - - case 122: - w7[2] = w7[2] | 0x020000; - break; - - case 123: - w7[2] = w7[2] | 0x02000000; - break; - - case 124: - w7[3] = 0x02; - break; - - case 125: - w7[3] = w7[3] | 0x0200; - break; - - case 126: - w7[3] = w7[3] | 0x020000; - break; - - case 127: - w7[3] = w7[3] | 0x02000000; - break; - } -} - -inline void append_0x80_1x4 (u32x w0[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x80; - break; - - case 1: - w0[0] = w0[0] | 0x8000; - break; - - case 2: - w0[0] = w0[0] | 0x800000; - break; - - case 3: - w0[0] = w0[0] | 0x80000000; - break; - - case 4: - w0[1] = 0x80; - break; - - case 5: - w0[1] = w0[1] | 0x8000; - break; - - case 6: - w0[1] = w0[1] | 0x800000; - break; - - case 7: - w0[1] = w0[1] | 0x80000000; - break; - - case 8: - w0[2] = 0x80; - break; - - case 9: - w0[2] = w0[2] | 0x8000; - break; - - case 10: - w0[2] = w0[2] | 0x800000; - break; - - case 11: - w0[2] = w0[2] | 0x80000000; - break; - - case 12: - w0[3] = 0x80; - break; - - case 13: - w0[3] = w0[3] | 0x8000; - break; - - case 14: - w0[3] = w0[3] | 0x800000; - break; - - case 15: - w0[3] = w0[3] | 0x80000000; - break; - } -} - -inline void append_0x80_2x4 (u32x w0[4], u32x w1[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x80; - break; - - case 1: - w0[0] = w0[0] | 0x8000; - break; - - case 2: - w0[0] = w0[0] | 0x800000; - break; - - case 3: - w0[0] = w0[0] | 0x80000000; - break; - - case 4: - w0[1] = 0x80; - break; - - case 5: - w0[1] = w0[1] | 0x8000; - break; - - case 6: - w0[1] = w0[1] | 0x800000; - break; - - case 7: - w0[1] = w0[1] | 0x80000000; - break; - - case 8: - w0[2] = 0x80; - break; - - case 9: - w0[2] = w0[2] | 0x8000; - break; - - case 10: - w0[2] = w0[2] | 0x800000; - break; - - case 11: - w0[2] = w0[2] | 0x80000000; - break; - - case 12: - w0[3] = 0x80; - break; - - case 13: - w0[3] = w0[3] | 0x8000; - break; - - case 14: - w0[3] = w0[3] | 0x800000; - break; - - case 15: - w0[3] = w0[3] | 0x80000000; - break; - - case 16: - w1[0] = 0x80; - break; - - case 17: - w1[0] = w1[0] | 0x8000; - break; - - case 18: - w1[0] = w1[0] | 0x800000; - break; - - case 19: - w1[0] = w1[0] | 0x80000000; - break; - - case 20: - w1[1] = 0x80; - break; - - case 21: - w1[1] = w1[1] | 0x8000; - break; - - case 22: - w1[1] = w1[1] | 0x800000; - break; - - case 23: - w1[1] = w1[1] | 0x80000000; - break; - - case 24: - w1[2] = 0x80; - break; - - case 25: - w1[2] = w1[2] | 0x8000; - break; - - case 26: - w1[2] = w1[2] | 0x800000; - break; - - case 27: - w1[2] = w1[2] | 0x80000000; - break; - - case 28: - w1[3] = 0x80; - break; - - case 29: - w1[3] = w1[3] | 0x8000; - break; - - case 30: - w1[3] = w1[3] | 0x800000; - break; - - case 31: - w1[3] = w1[3] | 0x80000000; - break; - } -} - -inline void append_0x80_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x80; - break; - - case 1: - w0[0] = w0[0] | 0x8000; - break; - - case 2: - w0[0] = w0[0] | 0x800000; - break; - - case 3: - w0[0] = w0[0] | 0x80000000; - break; - - case 4: - w0[1] = 0x80; - break; - - case 5: - w0[1] = w0[1] | 0x8000; - break; - - case 6: - w0[1] = w0[1] | 0x800000; - break; - - case 7: - w0[1] = w0[1] | 0x80000000; - break; - - case 8: - w0[2] = 0x80; - break; - - case 9: - w0[2] = w0[2] | 0x8000; - break; - - case 10: - w0[2] = w0[2] | 0x800000; - break; - - case 11: - w0[2] = w0[2] | 0x80000000; - break; - - case 12: - w0[3] = 0x80; - break; - - case 13: - w0[3] = w0[3] | 0x8000; - break; - - case 14: - w0[3] = w0[3] | 0x800000; - break; - - case 15: - w0[3] = w0[3] | 0x80000000; - break; - - case 16: - w1[0] = 0x80; - break; - - case 17: - w1[0] = w1[0] | 0x8000; - break; - - case 18: - w1[0] = w1[0] | 0x800000; - break; - - case 19: - w1[0] = w1[0] | 0x80000000; - break; - - case 20: - w1[1] = 0x80; - break; - - case 21: - w1[1] = w1[1] | 0x8000; - break; - - case 22: - w1[1] = w1[1] | 0x800000; - break; - - case 23: - w1[1] = w1[1] | 0x80000000; - break; - - case 24: - w1[2] = 0x80; - break; - - case 25: - w1[2] = w1[2] | 0x8000; - break; - - case 26: - w1[2] = w1[2] | 0x800000; - break; - - case 27: - w1[2] = w1[2] | 0x80000000; - break; - - case 28: - w1[3] = 0x80; - break; - - case 29: - w1[3] = w1[3] | 0x8000; - break; - - case 30: - w1[3] = w1[3] | 0x800000; - break; - - case 31: - w1[3] = w1[3] | 0x80000000; - break; - - case 32: - w2[0] = 0x80; - break; - - case 33: - w2[0] = w2[0] | 0x8000; - break; - - case 34: - w2[0] = w2[0] | 0x800000; - break; - - case 35: - w2[0] = w2[0] | 0x80000000; - break; - - case 36: - w2[1] = 0x80; - break; - - case 37: - w2[1] = w2[1] | 0x8000; - break; - - case 38: - w2[1] = w2[1] | 0x800000; - break; - - case 39: - w2[1] = w2[1] | 0x80000000; - break; - - case 40: - w2[2] = 0x80; - break; - - case 41: - w2[2] = w2[2] | 0x8000; - break; - - case 42: - w2[2] = w2[2] | 0x800000; - break; - - case 43: - w2[2] = w2[2] | 0x80000000; - break; - - case 44: - w2[3] = 0x80; - break; - - case 45: - w2[3] = w2[3] | 0x8000; - break; - - case 46: - w2[3] = w2[3] | 0x800000; - break; - - case 47: - w2[3] = w2[3] | 0x80000000; - break; - } -} - -inline void append_0x80_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x80; - break; - - case 1: - w0[0] = w0[0] | 0x8000; - break; - - case 2: - w0[0] = w0[0] | 0x800000; - break; - - case 3: - w0[0] = w0[0] | 0x80000000; - break; - - case 4: - w0[1] = 0x80; - break; - - case 5: - w0[1] = w0[1] | 0x8000; - break; - - case 6: - w0[1] = w0[1] | 0x800000; - break; - - case 7: - w0[1] = w0[1] | 0x80000000; - break; - - case 8: - w0[2] = 0x80; - break; - - case 9: - w0[2] = w0[2] | 0x8000; - break; - - case 10: - w0[2] = w0[2] | 0x800000; - break; - - case 11: - w0[2] = w0[2] | 0x80000000; - break; - - case 12: - w0[3] = 0x80; - break; - - case 13: - w0[3] = w0[3] | 0x8000; - break; - - case 14: - w0[3] = w0[3] | 0x800000; - break; - - case 15: - w0[3] = w0[3] | 0x80000000; - break; - - case 16: - w1[0] = 0x80; - break; - - case 17: - w1[0] = w1[0] | 0x8000; - break; - - case 18: - w1[0] = w1[0] | 0x800000; - break; - - case 19: - w1[0] = w1[0] | 0x80000000; - break; - - case 20: - w1[1] = 0x80; - break; - - case 21: - w1[1] = w1[1] | 0x8000; - break; - - case 22: - w1[1] = w1[1] | 0x800000; - break; - - case 23: - w1[1] = w1[1] | 0x80000000; - break; - - case 24: - w1[2] = 0x80; - break; - - case 25: - w1[2] = w1[2] | 0x8000; - break; - - case 26: - w1[2] = w1[2] | 0x800000; - break; - - case 27: - w1[2] = w1[2] | 0x80000000; - break; - - case 28: - w1[3] = 0x80; - break; - - case 29: - w1[3] = w1[3] | 0x8000; - break; - - case 30: - w1[3] = w1[3] | 0x800000; - break; - - case 31: - w1[3] = w1[3] | 0x80000000; - break; - - case 32: - w2[0] = 0x80; - break; - - case 33: - w2[0] = w2[0] | 0x8000; - break; - - case 34: - w2[0] = w2[0] | 0x800000; - break; - - case 35: - w2[0] = w2[0] | 0x80000000; - break; - - case 36: - w2[1] = 0x80; - break; - - case 37: - w2[1] = w2[1] | 0x8000; - break; - - case 38: - w2[1] = w2[1] | 0x800000; - break; - - case 39: - w2[1] = w2[1] | 0x80000000; - break; - - case 40: - w2[2] = 0x80; - break; - - case 41: - w2[2] = w2[2] | 0x8000; - break; - - case 42: - w2[2] = w2[2] | 0x800000; - break; - - case 43: - w2[2] = w2[2] | 0x80000000; - break; - - case 44: - w2[3] = 0x80; - break; - - case 45: - w2[3] = w2[3] | 0x8000; - break; - - case 46: - w2[3] = w2[3] | 0x800000; - break; - - case 47: - w2[3] = w2[3] | 0x80000000; - break; - - case 48: - w3[0] = 0x80; - break; - - case 49: - w3[0] = w3[0] | 0x8000; - break; - - case 50: - w3[0] = w3[0] | 0x800000; - break; - - case 51: - w3[0] = w3[0] | 0x80000000; - break; - - case 52: - w3[1] = 0x80; - break; - - case 53: - w3[1] = w3[1] | 0x8000; - break; - - case 54: - w3[1] = w3[1] | 0x800000; - break; - - case 55: - w3[1] = w3[1] | 0x80000000; - break; - - case 56: - w3[2] = 0x80; - break; - - case 57: - w3[2] = w3[2] | 0x8000; - break; - - case 58: - w3[2] = w3[2] | 0x800000; - break; - - case 59: - w3[2] = w3[2] | 0x80000000; - break; - - case 60: - w3[3] = 0x80; - break; - - case 61: - w3[3] = w3[3] | 0x8000; - break; - - case 62: - w3[3] = w3[3] | 0x800000; - break; - - case 63: - w3[3] = w3[3] | 0x80000000; - break; - } -} - -inline void append_0x80_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x80; - break; - - case 1: - w0[0] = w0[0] | 0x8000; - break; - - case 2: - w0[0] = w0[0] | 0x800000; - break; - - case 3: - w0[0] = w0[0] | 0x80000000; - break; - - case 4: - w0[1] = 0x80; - break; - - case 5: - w0[1] = w0[1] | 0x8000; - break; - - case 6: - w0[1] = w0[1] | 0x800000; - break; - - case 7: - w0[1] = w0[1] | 0x80000000; - break; - - case 8: - w0[2] = 0x80; - break; - - case 9: - w0[2] = w0[2] | 0x8000; - break; - - case 10: - w0[2] = w0[2] | 0x800000; - break; - - case 11: - w0[2] = w0[2] | 0x80000000; - break; - - case 12: - w0[3] = 0x80; - break; - - case 13: - w0[3] = w0[3] | 0x8000; - break; - - case 14: - w0[3] = w0[3] | 0x800000; - break; - - case 15: - w0[3] = w0[3] | 0x80000000; - break; - - case 16: - w1[0] = 0x80; - break; - - case 17: - w1[0] = w1[0] | 0x8000; - break; - - case 18: - w1[0] = w1[0] | 0x800000; - break; - - case 19: - w1[0] = w1[0] | 0x80000000; - break; - - case 20: - w1[1] = 0x80; - break; - - case 21: - w1[1] = w1[1] | 0x8000; - break; - - case 22: - w1[1] = w1[1] | 0x800000; - break; - - case 23: - w1[1] = w1[1] | 0x80000000; - break; - - case 24: - w1[2] = 0x80; - break; - - case 25: - w1[2] = w1[2] | 0x8000; - break; - - case 26: - w1[2] = w1[2] | 0x800000; - break; - - case 27: - w1[2] = w1[2] | 0x80000000; - break; - - case 28: - w1[3] = 0x80; - break; - - case 29: - w1[3] = w1[3] | 0x8000; - break; - - case 30: - w1[3] = w1[3] | 0x800000; - break; - - case 31: - w1[3] = w1[3] | 0x80000000; - break; - - case 32: - w2[0] = 0x80; - break; - - case 33: - w2[0] = w2[0] | 0x8000; - break; - - case 34: - w2[0] = w2[0] | 0x800000; - break; - - case 35: - w2[0] = w2[0] | 0x80000000; - break; - - case 36: - w2[1] = 0x80; - break; - - case 37: - w2[1] = w2[1] | 0x8000; - break; - - case 38: - w2[1] = w2[1] | 0x800000; - break; - - case 39: - w2[1] = w2[1] | 0x80000000; - break; - - case 40: - w2[2] = 0x80; - break; - - case 41: - w2[2] = w2[2] | 0x8000; - break; - - case 42: - w2[2] = w2[2] | 0x800000; - break; - - case 43: - w2[2] = w2[2] | 0x80000000; - break; - - case 44: - w2[3] = 0x80; - break; - - case 45: - w2[3] = w2[3] | 0x8000; - break; - - case 46: - w2[3] = w2[3] | 0x800000; - break; - - case 47: - w2[3] = w2[3] | 0x80000000; - break; - - case 48: - w3[0] = 0x80; - break; - - case 49: - w3[0] = w3[0] | 0x8000; - break; - - case 50: - w3[0] = w3[0] | 0x800000; - break; - - case 51: - w3[0] = w3[0] | 0x80000000; - break; - - case 52: - w3[1] = 0x80; - break; - - case 53: - w3[1] = w3[1] | 0x8000; - break; - - case 54: - w3[1] = w3[1] | 0x800000; - break; - - case 55: - w3[1] = w3[1] | 0x80000000; - break; - - case 56: - w3[2] = 0x80; - break; - - case 57: - w3[2] = w3[2] | 0x8000; - break; - - case 58: - w3[2] = w3[2] | 0x800000; - break; - - case 59: - w3[2] = w3[2] | 0x80000000; - break; - - case 60: - w3[3] = 0x80; - break; - - case 61: - w3[3] = w3[3] | 0x8000; - break; - - case 62: - w3[3] = w3[3] | 0x800000; - break; - - case 63: - w3[3] = w3[3] | 0x80000000; - break; - - case 64: - w4[0] = 0x80; - break; - - case 65: - w4[0] = w4[0] | 0x8000; - break; - - case 66: - w4[0] = w4[0] | 0x800000; - break; - - case 67: - w4[0] = w4[0] | 0x80000000; - break; - - case 68: - w4[1] = 0x80; - break; - - case 69: - w4[1] = w4[1] | 0x8000; - break; - - case 70: - w4[1] = w4[1] | 0x800000; - break; - - case 71: - w4[1] = w4[1] | 0x80000000; - break; - - case 72: - w4[2] = 0x80; - break; - - case 73: - w4[2] = w4[2] | 0x8000; - break; - - case 74: - w4[2] = w4[2] | 0x800000; - break; - - case 75: - w4[2] = w4[2] | 0x80000000; - break; - - case 76: - w4[3] = 0x80; - break; - - case 77: - w4[3] = w4[3] | 0x8000; - break; - - case 78: - w4[3] = w4[3] | 0x800000; - break; - - case 79: - w4[3] = w4[3] | 0x80000000; - break; - - case 80: - w5[0] = 0x80; - break; - - case 81: - w5[0] = w5[0] | 0x8000; - break; - - case 82: - w5[0] = w5[0] | 0x800000; - break; - - case 83: - w5[0] = w5[0] | 0x80000000; - break; - - case 84: - w5[1] = 0x80; - break; - - case 85: - w5[1] = w5[1] | 0x8000; - break; - - case 86: - w5[1] = w5[1] | 0x800000; - break; - - case 87: - w5[1] = w5[1] | 0x80000000; - break; - - case 88: - w5[2] = 0x80; - break; - - case 89: - w5[2] = w5[2] | 0x8000; - break; - - case 90: - w5[2] = w5[2] | 0x800000; - break; - - case 91: - w5[2] = w5[2] | 0x80000000; - break; - - case 92: - w5[3] = 0x80; - break; - - case 93: - w5[3] = w5[3] | 0x8000; - break; - - case 94: - w5[3] = w5[3] | 0x800000; - break; - - case 95: - w5[3] = w5[3] | 0x80000000; - break; - - case 96: - w6[0] = 0x80; - break; - - case 97: - w6[0] = w6[0] | 0x8000; - break; - - case 98: - w6[0] = w6[0] | 0x800000; - break; - - case 99: - w6[0] = w6[0] | 0x80000000; - break; - - case 100: - w6[1] = 0x80; - break; - - case 101: - w6[1] = w6[1] | 0x8000; - break; - - case 102: - w6[1] = w6[1] | 0x800000; - break; - - case 103: - w6[1] = w6[1] | 0x80000000; - break; - - case 104: - w6[2] = 0x80; - break; - - case 105: - w6[2] = w6[2] | 0x8000; - break; - - case 106: - w6[2] = w6[2] | 0x800000; - break; - - case 107: - w6[2] = w6[2] | 0x80000000; - break; - - case 108: - w6[3] = 0x80; - break; - - case 109: - w6[3] = w6[3] | 0x8000; - break; - - case 110: - w6[3] = w6[3] | 0x800000; - break; - - case 111: - w6[3] = w6[3] | 0x80000000; - break; - - case 112: - w7[0] = 0x80; - break; - - case 113: - w7[0] = w7[0] | 0x8000; - break; - - case 114: - w7[0] = w7[0] | 0x800000; - break; - - case 115: - w7[0] = w7[0] | 0x80000000; - break; - - case 116: - w7[1] = 0x80; - break; - - case 117: - w7[1] = w7[1] | 0x8000; - break; - - case 118: - w7[1] = w7[1] | 0x800000; - break; - - case 119: - w7[1] = w7[1] | 0x80000000; - break; - - case 120: - w7[2] = 0x80; - break; - - case 121: - w7[2] = w7[2] | 0x8000; - break; - - case 122: - w7[2] = w7[2] | 0x800000; - break; - - case 123: - w7[2] = w7[2] | 0x80000000; - break; - - case 124: - w7[3] = 0x80; - break; - - case 125: - w7[3] = w7[3] | 0x8000; - break; - - case 126: - w7[3] = w7[3] | 0x800000; - break; - - case 127: - w7[3] = w7[3] | 0x80000000; - break; - } -} - -inline void append_0x80_1x16 (u32x w[16], const u32 offset) -{ - switch (offset) - { - case 0: - w[ 0] = 0x80; - break; - - case 1: - w[ 0] = w[ 0] | 0x8000; - break; - - case 2: - w[ 0] = w[ 0] | 0x800000; - break; - - case 3: - w[ 0] = w[ 0] | 0x80000000; - break; - - case 4: - w[ 1] = 0x80; - break; - - case 5: - w[ 1] = w[ 1] | 0x8000; - break; - - case 6: - w[ 1] = w[ 1] | 0x800000; - break; - - case 7: - w[ 1] = w[ 1] | 0x80000000; - break; - - case 8: - w[ 2] = 0x80; - break; - - case 9: - w[ 2] = w[ 2] | 0x8000; - break; - - case 10: - w[ 2] = w[ 2] | 0x800000; - break; - - case 11: - w[ 2] = w[ 2] | 0x80000000; - break; - - case 12: - w[ 3] = 0x80; - break; - - case 13: - w[ 3] = w[ 3] | 0x8000; - break; - - case 14: - w[ 3] = w[ 3] | 0x800000; - break; - - case 15: - w[ 3] = w[ 3] | 0x80000000; - break; - - case 16: - w[ 4] = 0x80; - break; - - case 17: - w[ 4] = w[ 4] | 0x8000; - break; - - case 18: - w[ 4] = w[ 4] | 0x800000; - break; - - case 19: - w[ 4] = w[ 4] | 0x80000000; - break; - - case 20: - w[ 5] = 0x80; - break; - - case 21: - w[ 5] = w[ 5] | 0x8000; - break; - - case 22: - w[ 5] = w[ 5] | 0x800000; - break; - - case 23: - w[ 5] = w[ 5] | 0x80000000; - break; - - case 24: - w[ 6] = 0x80; - break; - - case 25: - w[ 6] = w[ 6] | 0x8000; - break; - - case 26: - w[ 6] = w[ 6] | 0x800000; - break; - - case 27: - w[ 6] = w[ 6] | 0x80000000; - break; - - case 28: - w[ 7] = 0x80; - break; - - case 29: - w[ 7] = w[ 7] | 0x8000; - break; - - case 30: - w[ 7] = w[ 7] | 0x800000; - break; - - case 31: - w[ 7] = w[ 7] | 0x80000000; - break; - - case 32: - w[ 8] = 0x80; - break; - - case 33: - w[ 8] = w[ 8] | 0x8000; - break; - - case 34: - w[ 8] = w[ 8] | 0x800000; - break; - - case 35: - w[ 8] = w[ 8] | 0x80000000; - break; - - case 36: - w[ 9] = 0x80; - break; - - case 37: - w[ 9] = w[ 9] | 0x8000; - break; - - case 38: - w[ 9] = w[ 9] | 0x800000; - break; - - case 39: - w[ 9] = w[ 9] | 0x80000000; - break; - - case 40: - w[10] = 0x80; - break; - - case 41: - w[10] = w[10] | 0x8000; - break; - - case 42: - w[10] = w[10] | 0x800000; - break; - - case 43: - w[10] = w[10] | 0x80000000; - break; - - case 44: - w[11] = 0x80; - break; - - case 45: - w[11] = w[11] | 0x8000; - break; - - case 46: - w[11] = w[11] | 0x800000; - break; - - case 47: - w[11] = w[11] | 0x80000000; - break; - - case 48: - w[12] = 0x80; - break; - - case 49: - w[12] = w[12] | 0x8000; - break; - - case 50: - w[12] = w[12] | 0x800000; - break; - - case 51: - w[12] = w[12] | 0x80000000; - break; - - case 52: - w[13] = 0x80; - break; - - case 53: - w[13] = w[13] | 0x8000; - break; - - case 54: - w[13] = w[13] | 0x800000; - break; - - case 55: - w[13] = w[13] | 0x80000000; - break; - - case 56: - w[14] = 0x80; - break; - - case 57: - w[14] = w[14] | 0x8000; - break; - - case 58: - w[14] = w[14] | 0x800000; - break; - - case 59: - w[14] = w[14] | 0x80000000; - break; - - case 60: - w[15] = 0x80; - break; - - case 61: - w[15] = w[15] | 0x8000; - break; - - case 62: - w[15] = w[15] | 0x800000; - break; - - case 63: - w[15] = w[15] | 0x80000000; - break; - } -} - -inline void switch_buffer_by_offset_le (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) -{ - #if defined IS_AMD || defined IS_GENERIC - const int offset_mod_4 = offset & 3; - - const int offset_minus_4 = 4 - offset; - - switch (offset / 4) - { - case 0: - w3[2] = amd_bytealign ( 0, w3[1], offset_minus_4); - w3[1] = amd_bytealign (w3[1], w3[0], offset_minus_4); - w3[0] = amd_bytealign (w3[0], w2[3], offset_minus_4); - w2[3] = amd_bytealign (w2[3], w2[2], offset_minus_4); - w2[2] = amd_bytealign (w2[2], w2[1], offset_minus_4); - w2[1] = amd_bytealign (w2[1], w2[0], offset_minus_4); - w2[0] = amd_bytealign (w2[0], w1[3], offset_minus_4); - w1[3] = amd_bytealign (w1[3], w1[2], offset_minus_4); - w1[2] = amd_bytealign (w1[2], w1[1], offset_minus_4); - w1[1] = amd_bytealign (w1[1], w1[0], offset_minus_4); - w1[0] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w0[3] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w0[2] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w0[1] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w0[0] = amd_bytealign (w0[0], 0, offset_minus_4); - - if (offset_mod_4 == 0) - { - w0[0] = w0[1]; - w0[1] = w0[2]; - w0[2] = w0[3]; - w0[3] = w1[0]; - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 1: - w3[2] = amd_bytealign ( 0, w3[0], offset_minus_4); - w3[1] = amd_bytealign (w3[0], w2[3], offset_minus_4); - w3[0] = amd_bytealign (w2[3], w2[2], offset_minus_4); - w2[3] = amd_bytealign (w2[2], w2[1], offset_minus_4); - w2[2] = amd_bytealign (w2[1], w2[0], offset_minus_4); - w2[1] = amd_bytealign (w2[0], w1[3], offset_minus_4); - w2[0] = amd_bytealign (w1[3], w1[2], offset_minus_4); - w1[3] = amd_bytealign (w1[2], w1[1], offset_minus_4); - w1[2] = amd_bytealign (w1[1], w1[0], offset_minus_4); - w1[1] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w1[0] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w0[3] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w0[2] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w0[1] = amd_bytealign (w0[0], 0, offset_minus_4); - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w0[1] = w0[2]; - w0[2] = w0[3]; - w0[3] = w1[0]; - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 2: - w3[2] = amd_bytealign ( 0, w2[3], offset_minus_4); - w3[1] = amd_bytealign (w2[3], w2[2], offset_minus_4); - w3[0] = amd_bytealign (w2[2], w2[1], offset_minus_4); - w2[3] = amd_bytealign (w2[1], w2[0], offset_minus_4); - w2[2] = amd_bytealign (w2[0], w1[3], offset_minus_4); - w2[1] = amd_bytealign (w1[3], w1[2], offset_minus_4); - w2[0] = amd_bytealign (w1[2], w1[1], offset_minus_4); - w1[3] = amd_bytealign (w1[1], w1[0], offset_minus_4); - w1[2] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w1[1] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w1[0] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w0[3] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w0[2] = amd_bytealign (w0[0], 0, offset_minus_4); - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w0[2] = w0[3]; - w0[3] = w1[0]; - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 3: - w3[2] = amd_bytealign ( 0, w2[2], offset_minus_4); - w3[1] = amd_bytealign (w2[2], w2[1], offset_minus_4); - w3[0] = amd_bytealign (w2[1], w2[0], offset_minus_4); - w2[3] = amd_bytealign (w2[0], w1[3], offset_minus_4); - w2[2] = amd_bytealign (w1[3], w1[2], offset_minus_4); - w2[1] = amd_bytealign (w1[2], w1[1], offset_minus_4); - w2[0] = amd_bytealign (w1[1], w1[0], offset_minus_4); - w1[3] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w1[2] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w1[1] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w1[0] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w0[3] = amd_bytealign (w0[0], 0, offset_minus_4); - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w0[3] = w1[0]; - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 4: - w3[2] = amd_bytealign ( 0, w2[1], offset_minus_4); - w3[1] = amd_bytealign (w2[1], w2[0], offset_minus_4); - w3[0] = amd_bytealign (w2[0], w1[3], offset_minus_4); - w2[3] = amd_bytealign (w1[3], w1[2], offset_minus_4); - w2[2] = amd_bytealign (w1[2], w1[1], offset_minus_4); - w2[1] = amd_bytealign (w1[1], w1[0], offset_minus_4); - w2[0] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w1[3] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w1[2] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w1[1] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w1[0] = amd_bytealign (w0[0], 0, offset_minus_4); - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 5: - w3[2] = amd_bytealign ( 0, w2[0], offset_minus_4); - w3[1] = amd_bytealign (w2[0], w1[3], offset_minus_4); - w3[0] = amd_bytealign (w1[3], w1[2], offset_minus_4); - w2[3] = amd_bytealign (w1[2], w1[1], offset_minus_4); - w2[2] = amd_bytealign (w1[1], w1[0], offset_minus_4); - w2[1] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w2[0] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w1[3] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w1[2] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w1[1] = amd_bytealign (w0[0], 0, offset_minus_4); - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 6: - w3[2] = amd_bytealign ( 0, w1[3], offset_minus_4); - w3[1] = amd_bytealign (w1[3], w1[2], offset_minus_4); - w3[0] = amd_bytealign (w1[2], w1[1], offset_minus_4); - w2[3] = amd_bytealign (w1[1], w1[0], offset_minus_4); - w2[2] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w2[1] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w2[0] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w1[3] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w1[2] = amd_bytealign (w0[0], 0, offset_minus_4); - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 7: - w3[2] = amd_bytealign ( 0, w1[2], offset_minus_4); - w3[1] = amd_bytealign (w1[2], w1[1], offset_minus_4); - w3[0] = amd_bytealign (w1[1], w1[0], offset_minus_4); - w2[3] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w2[2] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w2[1] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w2[0] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w1[3] = amd_bytealign (w0[0], 0, offset_minus_4); - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 8: - w3[2] = amd_bytealign ( 0, w1[1], offset_minus_4); - w3[1] = amd_bytealign (w1[1], w1[0], offset_minus_4); - w3[0] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w2[3] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w2[2] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w2[1] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w2[0] = amd_bytealign (w0[0], 0, offset_minus_4); - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 9: - w3[2] = amd_bytealign ( 0, w1[0], offset_minus_4); - w3[1] = amd_bytealign (w1[0], w0[3], offset_minus_4); - w3[0] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w2[3] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w2[2] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w2[1] = amd_bytealign (w0[0], 0, offset_minus_4); - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 10: - w3[2] = amd_bytealign ( 0, w0[3], offset_minus_4); - w3[1] = amd_bytealign (w0[3], w0[2], offset_minus_4); - w3[0] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w2[3] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w2[2] = amd_bytealign (w0[0], 0, offset_minus_4); - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 11: - w3[2] = amd_bytealign ( 0, w0[2], offset_minus_4); - w3[1] = amd_bytealign (w0[2], w0[1], offset_minus_4); - w3[0] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w2[3] = amd_bytealign (w0[0], 0, offset_minus_4); - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 12: - w3[2] = amd_bytealign ( 0, w0[1], offset_minus_4); - w3[1] = amd_bytealign (w0[1], w0[0], offset_minus_4); - w3[0] = amd_bytealign (w0[0], 0, offset_minus_4); - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 13: - w3[2] = amd_bytealign ( 0, w0[0], offset_minus_4); - w3[1] = amd_bytealign (w0[0], 0, offset_minus_4); - w3[0] = 0; - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - } - #endif - - #ifdef IS_NV - const int offset_minus_4 = 4 - (offset % 4); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - switch (offset / 4) - { - case 0: - w3[1] = __byte_perm (w3[0], w3[1], selector); - w3[0] = __byte_perm (w2[3], w3[0], selector); - w2[3] = __byte_perm (w2[2], w2[3], selector); - w2[2] = __byte_perm (w2[1], w2[2], selector); - w2[1] = __byte_perm (w2[0], w2[1], selector); - w2[0] = __byte_perm (w1[3], w2[0], selector); - w1[3] = __byte_perm (w1[2], w1[3], selector); - w1[2] = __byte_perm (w1[1], w1[2], selector); - w1[1] = __byte_perm (w1[0], w1[1], selector); - w1[0] = __byte_perm (w0[3], w1[0], selector); - w0[3] = __byte_perm (w0[2], w0[3], selector); - w0[2] = __byte_perm (w0[1], w0[2], selector); - w0[1] = __byte_perm (w0[0], w0[1], selector); - w0[0] = __byte_perm ( 0, w0[0], selector); - - break; - - case 1: - w3[1] = __byte_perm (w2[3], w3[0], selector); - w3[0] = __byte_perm (w2[2], w2[3], selector); - w2[3] = __byte_perm (w2[1], w2[2], selector); - w2[2] = __byte_perm (w2[0], w2[1], selector); - w2[1] = __byte_perm (w1[3], w2[0], selector); - w2[0] = __byte_perm (w1[2], w1[3], selector); - w1[3] = __byte_perm (w1[1], w1[2], selector); - w1[2] = __byte_perm (w1[0], w1[1], selector); - w1[1] = __byte_perm (w0[3], w1[0], selector); - w1[0] = __byte_perm (w0[2], w0[3], selector); - w0[3] = __byte_perm (w0[1], w0[2], selector); - w0[2] = __byte_perm (w0[0], w0[1], selector); - w0[1] = __byte_perm ( 0, w0[0], selector); - w0[0] = 0; - - break; - - case 2: - w3[1] = __byte_perm (w2[2], w2[3], selector); - w3[0] = __byte_perm (w2[1], w2[2], selector); - w2[3] = __byte_perm (w2[0], w2[1], selector); - w2[2] = __byte_perm (w1[3], w2[0], selector); - w2[1] = __byte_perm (w1[2], w1[3], selector); - w2[0] = __byte_perm (w1[1], w1[2], selector); - w1[3] = __byte_perm (w1[0], w1[1], selector); - w1[2] = __byte_perm (w0[3], w1[0], selector); - w1[1] = __byte_perm (w0[2], w0[3], selector); - w1[0] = __byte_perm (w0[1], w0[2], selector); - w0[3] = __byte_perm (w0[0], w0[1], selector); - w0[2] = __byte_perm ( 0, w0[0], selector); - w0[1] = 0; - w0[0] = 0; - - break; - - case 3: - w3[1] = __byte_perm (w2[1], w2[2], selector); - w3[0] = __byte_perm (w2[0], w2[1], selector); - w2[3] = __byte_perm (w1[3], w2[0], selector); - w2[2] = __byte_perm (w1[2], w1[3], selector); - w2[1] = __byte_perm (w1[1], w1[2], selector); - w2[0] = __byte_perm (w1[0], w1[1], selector); - w1[3] = __byte_perm (w0[3], w1[0], selector); - w1[2] = __byte_perm (w0[2], w0[3], selector); - w1[1] = __byte_perm (w0[1], w0[2], selector); - w1[0] = __byte_perm (w0[0], w0[1], selector); - w0[3] = __byte_perm ( 0, w0[0], selector); - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 4: - w3[1] = __byte_perm (w2[0], w2[1], selector); - w3[0] = __byte_perm (w1[3], w2[0], selector); - w2[3] = __byte_perm (w1[2], w1[3], selector); - w2[2] = __byte_perm (w1[1], w1[2], selector); - w2[1] = __byte_perm (w1[0], w1[1], selector); - w2[0] = __byte_perm (w0[3], w1[0], selector); - w1[3] = __byte_perm (w0[2], w0[3], selector); - w1[2] = __byte_perm (w0[1], w0[2], selector); - w1[1] = __byte_perm (w0[0], w0[1], selector); - w1[0] = __byte_perm ( 0, w0[0], selector); - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 5: - w3[1] = __byte_perm (w1[3], w2[0], selector); - w3[0] = __byte_perm (w1[2], w1[3], selector); - w2[3] = __byte_perm (w1[1], w1[2], selector); - w2[2] = __byte_perm (w1[0], w1[1], selector); - w2[1] = __byte_perm (w0[3], w1[0], selector); - w2[0] = __byte_perm (w0[2], w0[3], selector); - w1[3] = __byte_perm (w0[1], w0[2], selector); - w1[2] = __byte_perm (w0[0], w0[1], selector); - w1[1] = __byte_perm ( 0, w0[0], selector); - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 6: - w3[1] = __byte_perm (w1[2], w1[3], selector); - w3[0] = __byte_perm (w1[1], w1[2], selector); - w2[3] = __byte_perm (w1[0], w1[1], selector); - w2[2] = __byte_perm (w0[3], w1[0], selector); - w2[1] = __byte_perm (w0[2], w0[3], selector); - w2[0] = __byte_perm (w0[1], w0[2], selector); - w1[3] = __byte_perm (w0[0], w0[1], selector); - w1[2] = __byte_perm ( 0, w0[0], selector); - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 7: - w3[1] = __byte_perm (w1[1], w1[2], selector); - w3[0] = __byte_perm (w1[0], w1[1], selector); - w2[3] = __byte_perm (w0[3], w1[0], selector); - w2[2] = __byte_perm (w0[2], w0[3], selector); - w2[1] = __byte_perm (w0[1], w0[2], selector); - w2[0] = __byte_perm (w0[0], w0[1], selector); - w1[3] = __byte_perm ( 0, w0[0], selector); - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 8: - w3[1] = __byte_perm (w1[0], w1[1], selector); - w3[0] = __byte_perm (w0[3], w1[0], selector); - w2[3] = __byte_perm (w0[2], w0[3], selector); - w2[2] = __byte_perm (w0[1], w0[2], selector); - w2[1] = __byte_perm (w0[0], w0[1], selector); - w2[0] = __byte_perm ( 0, w0[0], selector); - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 9: - w3[1] = __byte_perm (w0[3], w1[0], selector); - w3[0] = __byte_perm (w0[2], w0[3], selector); - w2[3] = __byte_perm (w0[1], w0[2], selector); - w2[2] = __byte_perm (w0[0], w0[1], selector); - w2[1] = __byte_perm ( 0, w0[0], selector); - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 10: - w3[1] = __byte_perm (w0[2], w0[3], selector); - w3[0] = __byte_perm (w0[1], w0[2], selector); - w2[3] = __byte_perm (w0[0], w0[1], selector); - w2[2] = __byte_perm ( 0, w0[0], selector); - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 11: - w3[1] = __byte_perm (w0[1], w0[2], selector); - w3[0] = __byte_perm (w0[0], w0[1], selector); - w2[3] = __byte_perm ( 0, w0[0], selector); - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 12: - w3[1] = __byte_perm (w0[0], w0[1], selector); - w3[0] = __byte_perm ( 0, w0[0], selector); - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 13: - w3[1] = __byte_perm ( 0, w0[0], selector); - w3[0] = 0; - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - } - #endif -} - -inline void switch_buffer_by_offset_be (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) -{ - #if defined IS_AMD || defined IS_GENERIC - switch (offset / 4) - { - case 0: - w3[2] = amd_bytealign (w3[1], 0, offset); - w3[1] = amd_bytealign (w3[0], w3[1], offset); - w3[0] = amd_bytealign (w2[3], w3[0], offset); - w2[3] = amd_bytealign (w2[2], w2[3], offset); - w2[2] = amd_bytealign (w2[1], w2[2], offset); - w2[1] = amd_bytealign (w2[0], w2[1], offset); - w2[0] = amd_bytealign (w1[3], w2[0], offset); - w1[3] = amd_bytealign (w1[2], w1[3], offset); - w1[2] = amd_bytealign (w1[1], w1[2], offset); - w1[1] = amd_bytealign (w1[0], w1[1], offset); - w1[0] = amd_bytealign (w0[3], w1[0], offset); - w0[3] = amd_bytealign (w0[2], w0[3], offset); - w0[2] = amd_bytealign (w0[1], w0[2], offset); - w0[1] = amd_bytealign (w0[0], w0[1], offset); - w0[0] = amd_bytealign ( 0, w0[0], offset); - break; - - case 1: - w3[2] = amd_bytealign (w3[0], 0, offset); - w3[1] = amd_bytealign (w2[3], w3[0], offset); - w3[0] = amd_bytealign (w2[2], w2[3], offset); - w2[3] = amd_bytealign (w2[1], w2[2], offset); - w2[2] = amd_bytealign (w2[0], w2[1], offset); - w2[1] = amd_bytealign (w1[3], w2[0], offset); - w2[0] = amd_bytealign (w1[2], w1[3], offset); - w1[3] = amd_bytealign (w1[1], w1[2], offset); - w1[2] = amd_bytealign (w1[0], w1[1], offset); - w1[1] = amd_bytealign (w0[3], w1[0], offset); - w1[0] = amd_bytealign (w0[2], w0[3], offset); - w0[3] = amd_bytealign (w0[1], w0[2], offset); - w0[2] = amd_bytealign (w0[0], w0[1], offset); - w0[1] = amd_bytealign ( 0, w0[0], offset); - w0[0] = 0; - break; - - case 2: - w3[2] = amd_bytealign (w2[3], 0, offset); - w3[1] = amd_bytealign (w2[2], w2[3], offset); - w3[0] = amd_bytealign (w2[1], w2[2], offset); - w2[3] = amd_bytealign (w2[0], w2[1], offset); - w2[2] = amd_bytealign (w1[3], w2[0], offset); - w2[1] = amd_bytealign (w1[2], w1[3], offset); - w2[0] = amd_bytealign (w1[1], w1[2], offset); - w1[3] = amd_bytealign (w1[0], w1[1], offset); - w1[2] = amd_bytealign (w0[3], w1[0], offset); - w1[1] = amd_bytealign (w0[2], w0[3], offset); - w1[0] = amd_bytealign (w0[1], w0[2], offset); - w0[3] = amd_bytealign (w0[0], w0[1], offset); - w0[2] = amd_bytealign ( 0, w0[0], offset); - w0[1] = 0; - w0[0] = 0; - break; - - case 3: - w3[2] = amd_bytealign (w2[2], 0, offset); - w3[1] = amd_bytealign (w2[1], w2[2], offset); - w3[0] = amd_bytealign (w2[0], w2[1], offset); - w2[3] = amd_bytealign (w1[3], w2[0], offset); - w2[2] = amd_bytealign (w1[2], w1[3], offset); - w2[1] = amd_bytealign (w1[1], w1[2], offset); - w2[0] = amd_bytealign (w1[0], w1[1], offset); - w1[3] = amd_bytealign (w0[3], w1[0], offset); - w1[2] = amd_bytealign (w0[2], w0[3], offset); - w1[1] = amd_bytealign (w0[1], w0[2], offset); - w1[0] = amd_bytealign (w0[0], w0[1], offset); - w0[3] = amd_bytealign ( 0, w0[0], offset); - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 4: - w3[2] = amd_bytealign (w2[1], 0, offset); - w3[1] = amd_bytealign (w2[0], w2[1], offset); - w3[0] = amd_bytealign (w1[3], w2[0], offset); - w2[3] = amd_bytealign (w1[2], w1[3], offset); - w2[2] = amd_bytealign (w1[1], w1[2], offset); - w2[1] = amd_bytealign (w1[0], w1[1], offset); - w2[0] = amd_bytealign (w0[3], w1[0], offset); - w1[3] = amd_bytealign (w0[2], w0[3], offset); - w1[2] = amd_bytealign (w0[1], w0[2], offset); - w1[1] = amd_bytealign (w0[0], w0[1], offset); - w1[0] = amd_bytealign ( 0, w0[0], offset); - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 5: - w3[2] = amd_bytealign (w2[0], 0, offset); - w3[1] = amd_bytealign (w1[3], w2[0], offset); - w3[0] = amd_bytealign (w1[2], w1[3], offset); - w2[3] = amd_bytealign (w1[1], w1[2], offset); - w2[2] = amd_bytealign (w1[0], w1[1], offset); - w2[1] = amd_bytealign (w0[3], w1[0], offset); - w2[0] = amd_bytealign (w0[2], w0[3], offset); - w1[3] = amd_bytealign (w0[1], w0[2], offset); - w1[2] = amd_bytealign (w0[0], w0[1], offset); - w1[1] = amd_bytealign ( 0, w0[0], offset); - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 6: - w3[2] = amd_bytealign (w1[3], 0, offset); - w3[1] = amd_bytealign (w1[2], w1[3], offset); - w3[0] = amd_bytealign (w1[1], w1[2], offset); - w2[3] = amd_bytealign (w1[0], w1[1], offset); - w2[2] = amd_bytealign (w0[3], w1[0], offset); - w2[1] = amd_bytealign (w0[2], w0[3], offset); - w2[0] = amd_bytealign (w0[1], w0[2], offset); - w1[3] = amd_bytealign (w0[0], w0[1], offset); - w1[2] = amd_bytealign ( 0, w0[0], offset); - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 7: - w3[2] = amd_bytealign (w1[2], 0, offset); - w3[1] = amd_bytealign (w1[1], w1[2], offset); - w3[0] = amd_bytealign (w1[0], w1[1], offset); - w2[3] = amd_bytealign (w0[3], w1[0], offset); - w2[2] = amd_bytealign (w0[2], w0[3], offset); - w2[1] = amd_bytealign (w0[1], w0[2], offset); - w2[0] = amd_bytealign (w0[0], w0[1], offset); - w1[3] = amd_bytealign ( 0, w0[0], offset); - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 8: - w3[2] = amd_bytealign (w1[1], 0, offset); - w3[1] = amd_bytealign (w1[0], w1[1], offset); - w3[0] = amd_bytealign (w0[3], w1[0], offset); - w2[3] = amd_bytealign (w0[2], w0[3], offset); - w2[2] = amd_bytealign (w0[1], w0[2], offset); - w2[1] = amd_bytealign (w0[0], w0[1], offset); - w2[0] = amd_bytealign ( 0, w0[0], offset); - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 9: - w3[2] = amd_bytealign (w1[0], 0, offset); - w3[1] = amd_bytealign (w0[3], w1[0], offset); - w3[0] = amd_bytealign (w0[2], w0[3], offset); - w2[3] = amd_bytealign (w0[1], w0[2], offset); - w2[2] = amd_bytealign (w0[0], w0[1], offset); - w2[1] = amd_bytealign ( 0, w0[0], offset); - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 10: - w3[2] = amd_bytealign (w0[3], 0, offset); - w3[1] = amd_bytealign (w0[2], w0[3], offset); - w3[0] = amd_bytealign (w0[1], w0[2], offset); - w2[3] = amd_bytealign (w0[0], w0[1], offset); - w2[2] = amd_bytealign ( 0, w0[0], offset); - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 11: - w3[2] = amd_bytealign (w0[2], 0, offset); - w3[1] = amd_bytealign (w0[1], w0[2], offset); - w3[0] = amd_bytealign (w0[0], w0[1], offset); - w2[3] = amd_bytealign ( 0, w0[0], offset); - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 12: - w3[2] = amd_bytealign (w0[1], 0, offset); - w3[1] = amd_bytealign (w0[0], w0[1], offset); - w3[0] = amd_bytealign ( 0, w0[0], offset); - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 13: - w3[2] = amd_bytealign (w0[0], 0, offset); - w3[1] = amd_bytealign ( 0, w0[0], offset); - w3[0] = 0; - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - } - #endif - - #ifdef IS_NV - const int selector = (0x76543210 >> ((offset & 3) * 4)) & 0xffff; - - switch (offset / 4) - { - case 0: - w3[1] = __byte_perm (w3[1], w3[0], selector); - w3[0] = __byte_perm (w3[0], w2[3], selector); - w2[3] = __byte_perm (w2[3], w2[2], selector); - w2[2] = __byte_perm (w2[2], w2[1], selector); - w2[1] = __byte_perm (w2[1], w2[0], selector); - w2[0] = __byte_perm (w2[0], w1[3], selector); - w1[3] = __byte_perm (w1[3], w1[2], selector); - w1[2] = __byte_perm (w1[2], w1[1], selector); - w1[1] = __byte_perm (w1[1], w1[0], selector); - w1[0] = __byte_perm (w1[0], w0[3], selector); - w0[3] = __byte_perm (w0[3], w0[2], selector); - w0[2] = __byte_perm (w0[2], w0[1], selector); - w0[1] = __byte_perm (w0[1], w0[0], selector); - w0[0] = __byte_perm (w0[0], 0, selector); - break; - - case 1: - w3[1] = __byte_perm (w3[0], w2[3], selector); - w3[0] = __byte_perm (w2[3], w2[2], selector); - w2[3] = __byte_perm (w2[2], w2[1], selector); - w2[2] = __byte_perm (w2[1], w2[0], selector); - w2[1] = __byte_perm (w2[0], w1[3], selector); - w2[0] = __byte_perm (w1[3], w1[2], selector); - w1[3] = __byte_perm (w1[2], w1[1], selector); - w1[2] = __byte_perm (w1[1], w1[0], selector); - w1[1] = __byte_perm (w1[0], w0[3], selector); - w1[0] = __byte_perm (w0[3], w0[2], selector); - w0[3] = __byte_perm (w0[2], w0[1], selector); - w0[2] = __byte_perm (w0[1], w0[0], selector); - w0[1] = __byte_perm (w0[0], 0, selector); - w0[0] = 0; - break; - - case 2: - w3[1] = __byte_perm (w2[3], w2[2], selector); - w3[0] = __byte_perm (w2[2], w2[1], selector); - w2[3] = __byte_perm (w2[1], w2[0], selector); - w2[2] = __byte_perm (w2[0], w1[3], selector); - w2[1] = __byte_perm (w1[3], w1[2], selector); - w2[0] = __byte_perm (w1[2], w1[1], selector); - w1[3] = __byte_perm (w1[1], w1[0], selector); - w1[2] = __byte_perm (w1[0], w0[3], selector); - w1[1] = __byte_perm (w0[3], w0[2], selector); - w1[0] = __byte_perm (w0[2], w0[1], selector); - w0[3] = __byte_perm (w0[1], w0[0], selector); - w0[2] = __byte_perm (w0[0], 0, selector); - w0[1] = 0; - w0[0] = 0; - break; - - case 3: - w3[1] = __byte_perm (w2[2], w2[1], selector); - w3[0] = __byte_perm (w2[1], w2[0], selector); - w2[3] = __byte_perm (w2[0], w1[3], selector); - w2[2] = __byte_perm (w1[3], w1[2], selector); - w2[1] = __byte_perm (w1[2], w1[1], selector); - w2[0] = __byte_perm (w1[1], w1[0], selector); - w1[3] = __byte_perm (w1[0], w0[3], selector); - w1[2] = __byte_perm (w0[3], w0[2], selector); - w1[1] = __byte_perm (w0[2], w0[1], selector); - w1[0] = __byte_perm (w0[1], w0[0], selector); - w0[3] = __byte_perm (w0[0], 0, selector); - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 4: - w3[1] = __byte_perm (w2[1], w2[0], selector); - w3[0] = __byte_perm (w2[0], w1[3], selector); - w2[3] = __byte_perm (w1[3], w1[2], selector); - w2[2] = __byte_perm (w1[2], w1[1], selector); - w2[1] = __byte_perm (w1[1], w1[0], selector); - w2[0] = __byte_perm (w1[0], w0[3], selector); - w1[3] = __byte_perm (w0[3], w0[2], selector); - w1[2] = __byte_perm (w0[2], w0[1], selector); - w1[1] = __byte_perm (w0[1], w0[0], selector); - w1[0] = __byte_perm (w0[0], 0, selector); - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 5: - w3[1] = __byte_perm (w2[0], w1[3], selector); - w3[0] = __byte_perm (w1[3], w1[2], selector); - w2[3] = __byte_perm (w1[2], w1[1], selector); - w2[2] = __byte_perm (w1[1], w1[0], selector); - w2[1] = __byte_perm (w1[0], w0[3], selector); - w2[0] = __byte_perm (w0[3], w0[2], selector); - w1[3] = __byte_perm (w0[2], w0[1], selector); - w1[2] = __byte_perm (w0[1], w0[0], selector); - w1[1] = __byte_perm (w0[0], 0, selector); - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 6: - w3[1] = __byte_perm (w1[3], w1[2], selector); - w3[0] = __byte_perm (w1[2], w1[1], selector); - w2[3] = __byte_perm (w1[1], w1[0], selector); - w2[2] = __byte_perm (w1[0], w0[3], selector); - w2[1] = __byte_perm (w0[3], w0[2], selector); - w2[0] = __byte_perm (w0[2], w0[1], selector); - w1[3] = __byte_perm (w0[1], w0[0], selector); - w1[2] = __byte_perm (w0[0], 0, selector); - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 7: - w3[1] = __byte_perm (w1[2], w1[1], selector); - w3[0] = __byte_perm (w1[1], w1[0], selector); - w2[3] = __byte_perm (w1[0], w0[3], selector); - w2[2] = __byte_perm (w0[3], w0[2], selector); - w2[1] = __byte_perm (w0[2], w0[1], selector); - w2[0] = __byte_perm (w0[1], w0[0], selector); - w1[3] = __byte_perm (w0[0], 0, selector); - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 8: - w3[1] = __byte_perm (w1[1], w1[0], selector); - w3[0] = __byte_perm (w1[0], w0[3], selector); - w2[3] = __byte_perm (w0[3], w0[2], selector); - w2[2] = __byte_perm (w0[2], w0[1], selector); - w2[1] = __byte_perm (w0[1], w0[0], selector); - w2[0] = __byte_perm (w0[0], 0, selector); - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 9: - w3[1] = __byte_perm (w1[0], w0[3], selector); - w3[0] = __byte_perm (w0[3], w0[2], selector); - w2[3] = __byte_perm (w0[2], w0[1], selector); - w2[2] = __byte_perm (w0[1], w0[0], selector); - w2[1] = __byte_perm (w0[0], 0, selector); - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 10: - w3[1] = __byte_perm (w0[3], w0[2], selector); - w3[0] = __byte_perm (w0[2], w0[1], selector); - w2[3] = __byte_perm (w0[1], w0[0], selector); - w2[2] = __byte_perm (w0[0], 0, selector); - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 11: - w3[1] = __byte_perm (w0[2], w0[1], selector); - w3[0] = __byte_perm (w0[1], w0[0], selector); - w2[3] = __byte_perm (w0[0], 0, selector); - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 12: - w3[1] = __byte_perm (w0[1], w0[0], selector); - w3[0] = __byte_perm (w0[0], 0, selector); - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 13: - w3[1] = __byte_perm (w0[0], 0, selector); - w3[0] = 0; - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - } - #endif -} - -inline void overwrite_at_le (u32x sw[16], const u32x w0, const u32 salt_len) -{ - #if defined cl_amd_media_ops - switch (salt_len) - { - case 0: sw[0] = w0; - break; - case 1: sw[0] = amd_bytealign (w0, sw[0] << 24, 3); - sw[1] = amd_bytealign (sw[1] >> 8, w0, 3); - break; - case 2: sw[0] = amd_bytealign (w0, sw[0] << 16, 2); - sw[1] = amd_bytealign (sw[1] >> 16, w0, 2); - break; - case 3: sw[0] = amd_bytealign (w0, sw[0] << 8, 1); - sw[1] = amd_bytealign (sw[1] >> 24, w0, 1); - break; - case 4: sw[1] = w0; - break; - case 5: sw[1] = amd_bytealign (w0, sw[1] << 24, 3); - sw[2] = amd_bytealign (sw[2] >> 8, w0, 3); - break; - case 6: sw[1] = amd_bytealign (w0, sw[1] << 16, 2); - sw[2] = amd_bytealign (sw[2] >> 16, w0, 2); - break; - case 7: sw[1] = amd_bytealign (w0, sw[1] << 8, 1); - sw[2] = amd_bytealign (sw[2] >> 24, w0, 1); - break; - case 8: sw[2] = w0; - break; - case 9: sw[2] = amd_bytealign (w0, sw[2] << 24, 3); - sw[3] = amd_bytealign (sw[3] >> 8, w0, 3); - break; - case 10: sw[2] = amd_bytealign (w0, sw[2] << 16, 2); - sw[3] = amd_bytealign (sw[3] >> 16, w0, 2); - break; - case 11: sw[2] = amd_bytealign (w0, sw[2] << 8, 1); - sw[3] = amd_bytealign (sw[3] >> 24, w0, 1); - break; - case 12: sw[3] = w0; - break; - case 13: sw[3] = amd_bytealign (w0, sw[3] << 24, 3); - sw[4] = amd_bytealign (sw[4] >> 8, w0, 3); - break; - case 14: sw[3] = amd_bytealign (w0, sw[3] << 16, 2); - sw[4] = amd_bytealign (sw[4] >> 16, w0, 2); - break; - case 15: sw[3] = amd_bytealign (w0, sw[3] << 8, 1); - sw[4] = amd_bytealign (sw[4] >> 24, w0, 1); - break; - case 16: sw[4] = w0; - break; - case 17: sw[4] = amd_bytealign (w0, sw[4] << 24, 3); - sw[5] = amd_bytealign (sw[5] >> 8, w0, 3); - break; - case 18: sw[4] = amd_bytealign (w0, sw[4] << 16, 2); - sw[5] = amd_bytealign (sw[5] >> 16, w0, 2); - break; - case 19: sw[4] = amd_bytealign (w0, sw[4] << 8, 1); - sw[5] = amd_bytealign (sw[5] >> 24, w0, 1); - break; - case 20: sw[5] = w0; - break; - case 21: sw[5] = amd_bytealign (w0, sw[5] << 24, 3); - sw[6] = amd_bytealign (sw[6] >> 8, w0, 3); - break; - case 22: sw[5] = amd_bytealign (w0, sw[5] << 16, 2); - sw[6] = amd_bytealign (sw[6] >> 16, w0, 2); - break; - case 23: sw[5] = amd_bytealign (w0, sw[5] << 8, 1); - sw[6] = amd_bytealign (sw[6] >> 24, w0, 1); - break; - case 24: sw[6] = w0; - break; - case 25: sw[6] = amd_bytealign (w0, sw[6] << 24, 3); - sw[7] = amd_bytealign (sw[7] >> 8, w0, 3); - break; - case 26: sw[6] = amd_bytealign (w0, sw[6] << 16, 2); - sw[7] = amd_bytealign (sw[7] >> 16, w0, 2); - break; - case 27: sw[6] = amd_bytealign (w0, sw[6] << 8, 1); - sw[7] = amd_bytealign (sw[7] >> 24, w0, 1); - break; - case 28: sw[7] = w0; - break; - case 29: sw[7] = amd_bytealign (w0, sw[7] << 24, 3); - sw[8] = amd_bytealign (sw[8] >> 8, w0, 3); - break; - case 30: sw[7] = amd_bytealign (w0, sw[7] << 16, 2); - sw[8] = amd_bytealign (sw[8] >> 16, w0, 2); - break; - case 31: sw[7] = amd_bytealign (w0, sw[7] << 8, 1); - sw[8] = amd_bytealign (sw[8] >> 24, w0, 1); - break; - } - #else - switch (salt_len) - { - case 0: sw[0] = w0; - break; - case 1: sw[0] = (sw[0] & 0x000000ff) | (w0 << 8); - sw[1] = (sw[1] & 0xffffff00) | (w0 >> 24); - break; - case 2: sw[0] = (sw[0] & 0x0000ffff) | (w0 << 16); - sw[1] = (sw[1] & 0xffff0000) | (w0 >> 16); - break; - case 3: sw[0] = (sw[0] & 0x00ffffff) | (w0 << 24); - sw[1] = (sw[1] & 0xff000000) | (w0 >> 8); - break; - case 4: sw[1] = w0; - break; - case 5: sw[1] = (sw[1] & 0x000000ff) | (w0 << 8); - sw[2] = (sw[2] & 0xffffff00) | (w0 >> 24); - break; - case 6: sw[1] = (sw[1] & 0x0000ffff) | (w0 << 16); - sw[2] = (sw[2] & 0xffff0000) | (w0 >> 16); - break; - case 7: sw[1] = (sw[1] & 0x00ffffff) | (w0 << 24); - sw[2] = (sw[2] & 0xff000000) | (w0 >> 8); - break; - case 8: sw[2] = w0; - break; - case 9: sw[2] = (sw[2] & 0x000000ff) | (w0 << 8); - sw[3] = (sw[3] & 0xffffff00) | (w0 >> 24); - break; - case 10: sw[2] = (sw[2] & 0x0000ffff) | (w0 << 16); - sw[3] = (sw[3] & 0xffff0000) | (w0 >> 16); - break; - case 11: sw[2] = (sw[2] & 0x00ffffff) | (w0 << 24); - sw[3] = (sw[3] & 0xff000000) | (w0 >> 8); - break; - case 12: sw[3] = w0; - break; - case 13: sw[3] = (sw[3] & 0x000000ff) | (w0 << 8); - sw[4] = (sw[4] & 0xffffff00) | (w0 >> 24); - break; - case 14: sw[3] = (sw[3] & 0x0000ffff) | (w0 << 16); - sw[4] = (sw[4] & 0xffff0000) | (w0 >> 16); - break; - case 15: sw[3] = (sw[3] & 0x00ffffff) | (w0 << 24); - sw[4] = (sw[4] & 0xff000000) | (w0 >> 8); - break; - case 16: sw[4] = w0; - break; - case 17: sw[4] = (sw[4] & 0x000000ff) | (w0 << 8); - sw[5] = (sw[5] & 0xffffff00) | (w0 >> 24); - break; - case 18: sw[4] = (sw[4] & 0x0000ffff) | (w0 << 16); - sw[5] = (sw[5] & 0xffff0000) | (w0 >> 16); - break; - case 19: sw[4] = (sw[4] & 0x00ffffff) | (w0 << 24); - sw[5] = (sw[5] & 0xff000000) | (w0 >> 8); - break; - case 20: sw[5] = w0; - break; - case 21: sw[5] = (sw[5] & 0x000000ff) | (w0 << 8); - sw[6] = (sw[6] & 0xffffff00) | (w0 >> 24); - break; - case 22: sw[5] = (sw[5] & 0x0000ffff) | (w0 << 16); - sw[6] = (sw[6] & 0xffff0000) | (w0 >> 16); - break; - case 23: sw[5] = (sw[5] & 0x00ffffff) | (w0 << 24); - sw[6] = (sw[6] & 0xff000000) | (w0 >> 8); - break; - case 24: sw[6] = w0; - break; - case 25: sw[6] = (sw[6] & 0x000000ff) | (w0 << 8); - sw[7] = (sw[7] & 0xffffff00) | (w0 >> 24); - break; - case 26: sw[6] = (sw[6] & 0x0000ffff) | (w0 << 16); - sw[7] = (sw[7] & 0xffff0000) | (w0 >> 16); - break; - case 27: sw[6] = (sw[6] & 0x00ffffff) | (w0 << 24); - sw[7] = (sw[7] & 0xff000000) | (w0 >> 8); - break; - case 28: sw[7] = w0; - break; - case 29: sw[7] = (sw[7] & 0x000000ff) | (w0 << 8); - sw[8] = (sw[8] & 0xffffff00) | (w0 >> 24); - break; - case 30: sw[7] = (sw[7] & 0x0000ffff) | (w0 << 16); - sw[8] = (sw[8] & 0xffff0000) | (w0 >> 16); - break; - case 31: sw[7] = (sw[7] & 0x00ffffff) | (w0 << 24); - sw[8] = (sw[8] & 0xff000000) | (w0 >> 8); - break; - } - #endif -} - -inline void overwrite_at_be (u32x sw[16], const u32x w0, const u32 salt_len) -{ - // would be nice to have optimization based on amd_bytealign as with _le counterpart - - switch (salt_len) - { - case 0: sw[0] = w0; - break; - case 1: sw[0] = (sw[0] & 0xff000000) | (w0 >> 8); - sw[1] = (sw[1] & 0x00ffffff) | (w0 << 24); - break; - case 2: sw[0] = (sw[0] & 0xffff0000) | (w0 >> 16); - sw[1] = (sw[1] & 0x0000ffff) | (w0 << 16); - break; - case 3: sw[0] = (sw[0] & 0xffffff00) | (w0 >> 24); - sw[1] = (sw[1] & 0x000000ff) | (w0 << 8); - break; - case 4: sw[1] = w0; - break; - case 5: sw[1] = (sw[1] & 0xff000000) | (w0 >> 8); - sw[2] = (sw[2] & 0x00ffffff) | (w0 << 24); - break; - case 6: sw[1] = (sw[1] & 0xffff0000) | (w0 >> 16); - sw[2] = (sw[2] & 0x0000ffff) | (w0 << 16); - break; - case 7: sw[1] = (sw[1] & 0xffffff00) | (w0 >> 24); - sw[2] = (sw[2] & 0x000000ff) | (w0 << 8); - break; - case 8: sw[2] = w0; - break; - case 9: sw[2] = (sw[2] & 0xff000000) | (w0 >> 8); - sw[3] = (sw[3] & 0x00ffffff) | (w0 << 24); - break; - case 10: sw[2] = (sw[2] & 0xffff0000) | (w0 >> 16); - sw[3] = (sw[3] & 0x0000ffff) | (w0 << 16); - break; - case 11: sw[2] = (sw[2] & 0xffffff00) | (w0 >> 24); - sw[3] = (sw[3] & 0x000000ff) | (w0 << 8); - break; - case 12: sw[3] = w0; - break; - case 13: sw[3] = (sw[3] & 0xff000000) | (w0 >> 8); - sw[4] = (sw[4] & 0x00ffffff) | (w0 << 24); - break; - case 14: sw[3] = (sw[3] & 0xffff0000) | (w0 >> 16); - sw[4] = (sw[4] & 0x0000ffff) | (w0 << 16); - break; - case 15: sw[3] = (sw[3] & 0xffffff00) | (w0 >> 24); - sw[4] = (sw[4] & 0x000000ff) | (w0 << 8); - break; - case 16: sw[4] = w0; - break; - case 17: sw[4] = (sw[4] & 0xff000000) | (w0 >> 8); - sw[5] = (sw[5] & 0x00ffffff) | (w0 << 24); - break; - case 18: sw[4] = (sw[4] & 0xffff0000) | (w0 >> 16); - sw[5] = (sw[5] & 0x0000ffff) | (w0 << 16); - break; - case 19: sw[4] = (sw[4] & 0xffffff00) | (w0 >> 24); - sw[5] = (sw[5] & 0x000000ff) | (w0 << 8); - break; - case 20: sw[5] = w0; - break; - case 21: sw[5] = (sw[5] & 0xff000000) | (w0 >> 8); - sw[6] = (sw[6] & 0x00ffffff) | (w0 << 24); - break; - case 22: sw[5] = (sw[5] & 0xffff0000) | (w0 >> 16); - sw[6] = (sw[6] & 0x0000ffff) | (w0 << 16); - break; - case 23: sw[5] = (sw[5] & 0xffffff00) | (w0 >> 24); - sw[6] = (sw[6] & 0x000000ff) | (w0 << 8); - break; - case 24: sw[6] = w0; - break; - case 25: sw[6] = (sw[6] & 0xff000000) | (w0 >> 8); - sw[7] = (sw[7] & 0x00ffffff) | (w0 << 24); - break; - case 26: sw[6] = (sw[6] & 0xffff0000) | (w0 >> 16); - sw[7] = (sw[7] & 0x0000ffff) | (w0 << 16); - break; - case 27: sw[6] = (sw[6] & 0xffffff00) | (w0 >> 24); - sw[7] = (sw[7] & 0x000000ff) | (w0 << 8); - break; - case 28: sw[7] = w0; - break; - case 29: sw[7] = (sw[7] & 0xff000000) | (w0 >> 8); - sw[8] = (sw[8] & 0x00ffffff) | (w0 << 24); - break; - case 30: sw[7] = (sw[7] & 0xffff0000) | (w0 >> 16); - sw[8] = (sw[8] & 0x0000ffff) | (w0 << 16); - break; - case 31: sw[7] = (sw[7] & 0xffffff00) | (w0 >> 24); - sw[8] = (sw[8] & 0x000000ff) | (w0 << 8); - break; - } -} - -inline void overwrite_at_le_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x wx, const u32 salt_len) -{ - #if defined cl_amd_media_ops - switch (salt_len) - { - case 0: w0[0] = wx; - break; - case 1: w0[0] = amd_bytealign (wx, w0[0] << 24, 3); - w0[1] = amd_bytealign (w0[1] >> 8, wx, 3); - break; - case 2: w0[0] = amd_bytealign (wx, w0[0] << 16, 2); - w0[1] = amd_bytealign (w0[1] >> 16, wx, 2); - break; - case 3: w0[0] = amd_bytealign (wx, w0[0] << 8, 1); - w0[1] = amd_bytealign (w0[1] >> 24, wx, 1); - break; - case 4: w0[1] = wx; - break; - case 5: w0[1] = amd_bytealign (wx, w0[1] << 24, 3); - w0[2] = amd_bytealign (w0[2] >> 8, wx, 3); - break; - case 6: w0[1] = amd_bytealign (wx, w0[1] << 16, 2); - w0[2] = amd_bytealign (w0[2] >> 16, wx, 2); - break; - case 7: w0[1] = amd_bytealign (wx, w0[1] << 8, 1); - w0[2] = amd_bytealign (w0[2] >> 24, wx, 1); - break; - case 8: w0[2] = wx; - break; - case 9: w0[2] = amd_bytealign (wx, w0[2] << 24, 3); - w0[3] = amd_bytealign (w0[3] >> 8, wx, 3); - break; - case 10: w0[2] = amd_bytealign (wx, w0[2] << 16, 2); - w0[3] = amd_bytealign (w0[3] >> 16, wx, 2); - break; - case 11: w0[2] = amd_bytealign (wx, w0[2] << 8, 1); - w0[3] = amd_bytealign (w0[3] >> 24, wx, 1); - break; - case 12: w0[3] = wx; - break; - case 13: w0[3] = amd_bytealign (wx, w0[3] << 24, 3); - w1[0] = amd_bytealign (w1[0] >> 8, wx, 3); - break; - case 14: w0[3] = amd_bytealign (wx, w0[3] << 16, 2); - w1[0] = amd_bytealign (w1[0] >> 16, wx, 2); - break; - case 15: w0[3] = amd_bytealign (wx, w0[3] << 8, 1); - w1[0] = amd_bytealign (w1[0] >> 24, wx, 1); - break; - case 16: w1[0] = wx; - break; - case 17: w1[0] = amd_bytealign (wx, w1[0] << 24, 3); - w1[1] = amd_bytealign (w1[1] >> 8, wx, 3); - break; - case 18: w1[0] = amd_bytealign (wx, w1[0] << 16, 2); - w1[1] = amd_bytealign (w1[1] >> 16, wx, 2); - break; - case 19: w1[0] = amd_bytealign (wx, w1[0] << 8, 1); - w1[1] = amd_bytealign (w1[1] >> 24, wx, 1); - break; - case 20: w1[1] = wx; - break; - case 21: w1[1] = amd_bytealign (wx, w1[1] << 24, 3); - w1[2] = amd_bytealign (w1[2] >> 8, wx, 3); - break; - case 22: w1[1] = amd_bytealign (wx, w1[1] << 16, 2); - w1[2] = amd_bytealign (w1[2] >> 16, wx, 2); - break; - case 23: w1[1] = amd_bytealign (wx, w1[1] << 8, 1); - w1[2] = amd_bytealign (w1[2] >> 24, wx, 1); - break; - case 24: w1[2] = wx; - break; - case 25: w1[2] = amd_bytealign (wx, w1[2] << 24, 3); - w1[3] = amd_bytealign (w1[3] >> 8, wx, 3); - break; - case 26: w1[2] = amd_bytealign (wx, w1[2] << 16, 2); - w1[3] = amd_bytealign (w1[3] >> 16, wx, 2); - break; - case 27: w1[2] = amd_bytealign (wx, w1[2] << 8, 1); - w1[3] = amd_bytealign (w1[3] >> 24, wx, 1); - break; - case 28: w1[3] = wx; - break; - case 29: w1[3] = amd_bytealign (wx, w1[3] << 24, 3); - w2[0] = amd_bytealign (w2[0] >> 8, wx, 3); - break; - case 30: w1[3] = amd_bytealign (wx, w1[3] << 16, 2); - w2[0] = amd_bytealign (w2[0] >> 16, wx, 2); - break; - case 31: w1[3] = amd_bytealign (wx, w1[3] << 8, 1); - w2[0] = amd_bytealign (w2[0] >> 24, wx, 1); - break; - case 32: w2[0] = wx; - break; - case 33: w2[0] = amd_bytealign (wx, w2[0] << 24, 3); - w2[1] = amd_bytealign (w2[1] >> 8, wx, 3); - break; - case 34: w2[0] = amd_bytealign (wx, w2[0] << 16, 2); - w2[1] = amd_bytealign (w2[1] >> 16, wx, 2); - break; - case 35: w2[0] = amd_bytealign (wx, w2[0] << 8, 1); - w2[1] = amd_bytealign (w2[1] >> 24, wx, 1); - break; - case 36: w2[1] = wx; - break; - case 37: w2[1] = amd_bytealign (wx, w2[1] << 24, 3); - w2[2] = amd_bytealign (w2[2] >> 8, wx, 3); - break; - case 38: w2[1] = amd_bytealign (wx, w2[1] << 16, 2); - w2[2] = amd_bytealign (w2[2] >> 16, wx, 2); - break; - case 39: w2[1] = amd_bytealign (wx, w2[1] << 8, 1); - w2[2] = amd_bytealign (w2[2] >> 24, wx, 1); - break; - case 40: w2[2] = wx; - break; - case 41: w2[2] = amd_bytealign (wx, w2[2] << 24, 3); - w2[3] = amd_bytealign (w2[3] >> 8, wx, 3); - break; - case 42: w2[2] = amd_bytealign (wx, w2[2] << 16, 2); - w2[3] = amd_bytealign (w2[3] >> 16, wx, 2); - break; - case 43: w2[2] = amd_bytealign (wx, w2[2] << 8, 1); - w2[3] = amd_bytealign (w2[3] >> 24, wx, 1); - break; - case 44: w2[3] = wx; - break; - case 45: w2[3] = amd_bytealign (wx, w2[3] << 24, 3); - w3[0] = amd_bytealign (w3[0] >> 8, wx, 3); - break; - case 46: w2[3] = amd_bytealign (wx, w2[3] << 16, 2); - w3[0] = amd_bytealign (w3[0] >> 16, wx, 2); - break; - case 47: w2[3] = amd_bytealign (wx, w2[3] << 8, 1); - w3[0] = amd_bytealign (w3[0] >> 24, wx, 1); - break; - case 48: w3[0] = wx; - break; - case 49: w3[0] = amd_bytealign (wx, w3[0] << 24, 3); - w3[1] = amd_bytealign (w3[1] >> 8, wx, 3); - break; - case 50: w3[0] = amd_bytealign (wx, w3[0] << 16, 2); - w3[1] = amd_bytealign (w3[1] >> 16, wx, 2); - break; - case 51: w3[0] = amd_bytealign (wx, w3[0] << 8, 1); - w3[1] = amd_bytealign (w3[1] >> 24, wx, 1); - break; - case 52: w3[1] = wx; - break; - case 53: w3[1] = amd_bytealign (wx, w3[1] << 24, 3); - w3[2] = amd_bytealign (w3[2] >> 8, wx, 3); - break; - case 54: w3[1] = amd_bytealign (wx, w3[1] << 16, 2); - w3[2] = amd_bytealign (w3[2] >> 16, wx, 2); - break; - case 55: w3[1] = amd_bytealign (wx, w3[1] << 8, 1); - w3[2] = amd_bytealign (w3[2] >> 24, wx, 1); - break; - case 56: w3[2] = wx; - break; - case 57: w3[2] = amd_bytealign (wx, w3[2] << 24, 3); - w3[3] = amd_bytealign (w3[3] >> 8, wx, 3); - break; - case 58: w3[2] = amd_bytealign (wx, w3[2] << 16, 2); - w3[3] = amd_bytealign (w3[3] >> 16, wx, 2); - break; - case 59: w3[2] = amd_bytealign (wx, w3[2] << 8, 1); - w3[3] = amd_bytealign (w3[3] >> 24, wx, 1); - break; - case 60: w3[3] = wx; - break; - case 61: w3[3] = amd_bytealign (wx, w3[3] << 24, 3); - //w4[0] = amd_bytealign (w4[0] >> 8, wx, 3); - break; - case 62: w3[3] = amd_bytealign (wx, w3[3] << 16, 2); - //w4[0] = amd_bytealign (w4[0] >> 16, wx, 2); - break; - case 63: w3[3] = amd_bytealign (wx, w3[3] << 8, 1); - //w4[0] = amd_bytealign (w4[0] >> 24, wx, 1); - break; - } - #else - switch (salt_len) - { - case 0: w0[0] = wx; - break; - case 1: w0[0] = (w0[0] & 0x000000ff) | (wx << 8); - w0[1] = (w0[1] & 0xffffff00) | (wx >> 24); - break; - case 2: w0[0] = (w0[0] & 0x0000ffff) | (wx << 16); - w0[1] = (w0[1] & 0xffff0000) | (wx >> 16); - break; - case 3: w0[0] = (w0[0] & 0x00ffffff) | (wx << 24); - w0[1] = (w0[1] & 0xff000000) | (wx >> 8); - break; - case 4: w0[1] = wx; - break; - case 5: w0[1] = (w0[1] & 0x000000ff) | (wx << 8); - w0[2] = (w0[2] & 0xffffff00) | (wx >> 24); - break; - case 6: w0[1] = (w0[1] & 0x0000ffff) | (wx << 16); - w0[2] = (w0[2] & 0xffff0000) | (wx >> 16); - break; - case 7: w0[1] = (w0[1] & 0x00ffffff) | (wx << 24); - w0[2] = (w0[2] & 0xff000000) | (wx >> 8); - break; - case 8: w0[2] = wx; - break; - case 9: w0[2] = (w0[2] & 0x000000ff) | (wx << 8); - w0[3] = (w0[3] & 0xffffff00) | (wx >> 24); - break; - case 10: w0[2] = (w0[2] & 0x0000ffff) | (wx << 16); - w0[3] = (w0[3] & 0xffff0000) | (wx >> 16); - break; - case 11: w0[2] = (w0[2] & 0x00ffffff) | (wx << 24); - w0[3] = (w0[3] & 0xff000000) | (wx >> 8); - break; - case 12: w0[3] = wx; - break; - case 13: w0[3] = (w0[3] & 0x000000ff) | (wx << 8); - w1[0] = (w1[0] & 0xffffff00) | (wx >> 24); - break; - case 14: w0[3] = (w0[3] & 0x0000ffff) | (wx << 16); - w1[0] = (w1[0] & 0xffff0000) | (wx >> 16); - break; - case 15: w0[3] = (w0[3] & 0x00ffffff) | (wx << 24); - w1[0] = (w1[0] & 0xff000000) | (wx >> 8); - break; - case 16: w1[0] = wx; - break; - case 17: w1[0] = (w1[0] & 0x000000ff) | (wx << 8); - w1[1] = (w1[1] & 0xffffff00) | (wx >> 24); - break; - case 18: w1[0] = (w1[0] & 0x0000ffff) | (wx << 16); - w1[1] = (w1[1] & 0xffff0000) | (wx >> 16); - break; - case 19: w1[0] = (w1[0] & 0x00ffffff) | (wx << 24); - w1[1] = (w1[1] & 0xff000000) | (wx >> 8); - break; - case 20: w1[1] = wx; - break; - case 21: w1[1] = (w1[1] & 0x000000ff) | (wx << 8); - w1[2] = (w1[2] & 0xffffff00) | (wx >> 24); - break; - case 22: w1[1] = (w1[1] & 0x0000ffff) | (wx << 16); - w1[2] = (w1[2] & 0xffff0000) | (wx >> 16); - break; - case 23: w1[1] = (w1[1] & 0x00ffffff) | (wx << 24); - w1[2] = (w1[2] & 0xff000000) | (wx >> 8); - break; - case 24: w1[2] = wx; - break; - case 25: w1[2] = (w1[2] & 0x000000ff) | (wx << 8); - w1[3] = (w1[3] & 0xffffff00) | (wx >> 24); - break; - case 26: w1[2] = (w1[2] & 0x0000ffff) | (wx << 16); - w1[3] = (w1[3] & 0xffff0000) | (wx >> 16); - break; - case 27: w1[2] = (w1[2] & 0x00ffffff) | (wx << 24); - w1[3] = (w1[3] & 0xff000000) | (wx >> 8); - break; - case 28: w1[3] = wx; - break; - case 29: w1[3] = (w1[3] & 0x000000ff) | (wx << 8); - w2[0] = (w2[0] & 0xffffff00) | (wx >> 24); - break; - case 30: w1[3] = (w1[3] & 0x0000ffff) | (wx << 16); - w2[0] = (w2[0] & 0xffff0000) | (wx >> 16); - break; - case 31: w1[3] = (w1[3] & 0x00ffffff) | (wx << 24); - w2[0] = (w2[0] & 0xff000000) | (wx >> 8); - break; - case 32: w2[0] = wx; - break; - case 33: w2[0] = (w2[0] & 0x000000ff) | (wx << 8); - w2[1] = (w2[1] & 0xffffff00) | (wx >> 24); - break; - case 34: w2[0] = (w2[0] & 0x0000ffff) | (wx << 16); - w2[1] = (w2[1] & 0xffff0000) | (wx >> 16); - break; - case 35: w2[0] = (w2[0] & 0x00ffffff) | (wx << 24); - w2[1] = (w2[1] & 0xff000000) | (wx >> 8); - break; - case 36: w2[1] = wx; - break; - case 37: w2[1] = (w2[1] & 0x000000ff) | (wx << 8); - w2[2] = (w2[2] & 0xffffff00) | (wx >> 24); - break; - case 38: w2[1] = (w2[1] & 0x0000ffff) | (wx << 16); - w2[2] = (w2[2] & 0xffff0000) | (wx >> 16); - break; - case 39: w2[1] = (w2[1] & 0x00ffffff) | (wx << 24); - w2[2] = (w2[2] & 0xff000000) | (wx >> 8); - break; - case 40: w2[2] = wx; - break; - case 41: w2[2] = (w2[2] & 0x000000ff) | (wx << 8); - w2[3] = (w2[3] & 0xffffff00) | (wx >> 24); - break; - case 42: w2[2] = (w2[2] & 0x0000ffff) | (wx << 16); - w2[3] = (w2[3] & 0xffff0000) | (wx >> 16); - break; - case 43: w2[2] = (w2[2] & 0x00ffffff) | (wx << 24); - w2[3] = (w2[3] & 0xff000000) | (wx >> 8); - break; - case 44: w2[3] = wx; - break; - case 45: w2[3] = (w2[3] & 0x000000ff) | (wx << 8); - w3[0] = (w3[0] & 0xffffff00) | (wx >> 24); - break; - case 46: w2[3] = (w2[3] & 0x0000ffff) | (wx << 16); - w3[0] = (w3[0] & 0xffff0000) | (wx >> 16); - break; - case 47: w2[3] = (w2[3] & 0x00ffffff) | (wx << 24); - w3[0] = (w3[0] & 0xff000000) | (wx >> 8); - break; - case 48: w3[0] = wx; - break; - case 49: w3[0] = (w3[0] & 0x000000ff) | (wx << 8); - w3[1] = (w3[1] & 0xffffff00) | (wx >> 24); - break; - case 50: w3[0] = (w3[0] & 0x0000ffff) | (wx << 16); - w3[1] = (w3[1] & 0xffff0000) | (wx >> 16); - break; - case 51: w3[0] = (w3[0] & 0x00ffffff) | (wx << 24); - w3[1] = (w3[1] & 0xff000000) | (wx >> 8); - break; - case 52: w3[1] = wx; - break; - case 53: w3[1] = (w3[1] & 0x000000ff) | (wx << 8); - w3[2] = (w3[2] & 0xffffff00) | (wx >> 24); - break; - case 54: w3[1] = (w3[1] & 0x0000ffff) | (wx << 16); - w3[2] = (w3[2] & 0xffff0000) | (wx >> 16); - break; - case 55: w3[1] = (w3[1] & 0x00ffffff) | (wx << 24); - w3[2] = (w3[2] & 0xff000000) | (wx >> 8); - break; - case 56: w3[2] = wx; - break; - case 57: w3[2] = (w3[2] & 0x000000ff) | (wx << 8); - w3[3] = (w3[3] & 0xffffff00) | (wx >> 24); - break; - case 58: w3[2] = (w3[2] & 0x0000ffff) | (wx << 16); - w3[3] = (w3[3] & 0xffff0000) | (wx >> 16); - break; - case 59: w3[2] = (w3[2] & 0x00ffffff) | (wx << 24); - w3[3] = (w3[3] & 0xff000000) | (wx >> 8); - break; - case 60: w3[3] = wx; - break; - case 61: w3[3] = (w3[3] & 0x000000ff) | (wx << 8); - //w4[0] = (w4[0] & 0xffffff00) | (wx >> 24); - break; - case 62: w3[3] = (w3[3] & 0x0000ffff) | (wx << 16); - //w4[0] = (w4[0] & 0xffff0000) | (wx >> 16); - break; - case 63: w3[3] = (w3[3] & 0x00ffffff) | (wx << 24); - //w4[0] = (w4[0] & 0xff000000) | (wx >> 8); - break; - } - #endif -} - -inline void overwrite_at_be_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x wx, const u32 salt_len) -{ - // would be nice to have optimization based on amd_bytealign as with _le counterpart - - switch (salt_len) - { - case 0: w0[0] = wx; - break; - case 1: w0[0] = (w0[0] & 0xff000000) | (wx >> 8); - w0[1] = (w0[1] & 0x00ffffff) | (wx << 24); - break; - case 2: w0[0] = (w0[0] & 0xffff0000) | (wx >> 16); - w0[1] = (w0[1] & 0x0000ffff) | (wx << 16); - break; - case 3: w0[0] = (w0[0] & 0xffffff00) | (wx >> 24); - w0[1] = (w0[1] & 0x000000ff) | (wx << 8); - break; - case 4: w0[1] = wx; - break; - case 5: w0[1] = (w0[1] & 0xff000000) | (wx >> 8); - w0[2] = (w0[2] & 0x00ffffff) | (wx << 24); - break; - case 6: w0[1] = (w0[1] & 0xffff0000) | (wx >> 16); - w0[2] = (w0[2] & 0x0000ffff) | (wx << 16); - break; - case 7: w0[1] = (w0[1] & 0xffffff00) | (wx >> 24); - w0[2] = (w0[2] & 0x000000ff) | (wx << 8); - break; - case 8: w0[2] = wx; - break; - case 9: w0[2] = (w0[2] & 0xff000000) | (wx >> 8); - w0[3] = (w0[3] & 0x00ffffff) | (wx << 24); - break; - case 10: w0[2] = (w0[2] & 0xffff0000) | (wx >> 16); - w0[3] = (w0[3] & 0x0000ffff) | (wx << 16); - break; - case 11: w0[2] = (w0[2] & 0xffffff00) | (wx >> 24); - w0[3] = (w0[3] & 0x000000ff) | (wx << 8); - break; - case 12: w0[3] = wx; - break; - case 13: w0[3] = (w0[3] & 0xff000000) | (wx >> 8); - w1[0] = (w1[0] & 0x00ffffff) | (wx << 24); - break; - case 14: w0[3] = (w0[3] & 0xffff0000) | (wx >> 16); - w1[0] = (w1[0] & 0x0000ffff) | (wx << 16); - break; - case 15: w0[3] = (w0[3] & 0xffffff00) | (wx >> 24); - w1[0] = (w1[0] & 0x000000ff) | (wx << 8); - break; - case 16: w1[0] = wx; - break; - case 17: w1[0] = (w1[0] & 0xff000000) | (wx >> 8); - w1[1] = (w1[1] & 0x00ffffff) | (wx << 24); - break; - case 18: w1[0] = (w1[0] & 0xffff0000) | (wx >> 16); - w1[1] = (w1[1] & 0x0000ffff) | (wx << 16); - break; - case 19: w1[0] = (w1[0] & 0xffffff00) | (wx >> 24); - w1[1] = (w1[1] & 0x000000ff) | (wx << 8); - break; - case 20: w1[1] = wx; - break; - case 21: w1[1] = (w1[1] & 0xff000000) | (wx >> 8); - w1[2] = (w1[2] & 0x00ffffff) | (wx << 24); - break; - case 22: w1[1] = (w1[1] & 0xffff0000) | (wx >> 16); - w1[2] = (w1[2] & 0x0000ffff) | (wx << 16); - break; - case 23: w1[1] = (w1[1] & 0xffffff00) | (wx >> 24); - w1[2] = (w1[2] & 0x000000ff) | (wx << 8); - break; - case 24: w1[2] = wx; - break; - case 25: w1[2] = (w1[2] & 0xff000000) | (wx >> 8); - w1[3] = (w1[3] & 0x00ffffff) | (wx << 24); - break; - case 26: w1[2] = (w1[2] & 0xffff0000) | (wx >> 16); - w1[3] = (w1[3] & 0x0000ffff) | (wx << 16); - break; - case 27: w1[2] = (w1[2] & 0xffffff00) | (wx >> 24); - w1[3] = (w1[3] & 0x000000ff) | (wx << 8); - break; - case 28: w1[3] = wx; - break; - case 29: w1[3] = (w1[3] & 0xff000000) | (wx >> 8); - w2[0] = (w2[0] & 0x00ffffff) | (wx << 24); - break; - case 30: w1[3] = (w1[3] & 0xffff0000) | (wx >> 16); - w2[0] = (w2[0] & 0x0000ffff) | (wx << 16); - break; - case 31: w1[3] = (w1[3] & 0xffffff00) | (wx >> 24); - w2[0] = (w2[0] & 0x000000ff) | (wx << 8); - break; - case 32: w2[0] = wx; - break; - case 33: w2[0] = (w2[0] & 0xff000000) | (wx >> 8); - w2[1] = (w2[1] & 0x00ffffff) | (wx << 24); - break; - case 34: w2[0] = (w2[0] & 0xffff0000) | (wx >> 16); - w2[1] = (w2[1] & 0x0000ffff) | (wx << 16); - break; - case 35: w2[0] = (w2[0] & 0xffffff00) | (wx >> 24); - w2[1] = (w2[1] & 0x000000ff) | (wx << 8); - break; - case 36: w2[1] = wx; - break; - case 37: w2[1] = (w2[1] & 0xff000000) | (wx >> 8); - w2[2] = (w2[2] & 0x00ffffff) | (wx << 24); - break; - case 38: w2[1] = (w2[1] & 0xffff0000) | (wx >> 16); - w2[2] = (w2[2] & 0x0000ffff) | (wx << 16); - break; - case 39: w2[1] = (w2[1] & 0xffffff00) | (wx >> 24); - w2[2] = (w2[2] & 0x000000ff) | (wx << 8); - break; - case 40: w2[2] = wx; - break; - case 41: w2[2] = (w2[2] & 0xff000000) | (wx >> 8); - w2[3] = (w2[3] & 0x00ffffff) | (wx << 24); - break; - case 42: w2[2] = (w2[2] & 0xffff0000) | (wx >> 16); - w2[3] = (w2[3] & 0x0000ffff) | (wx << 16); - break; - case 43: w2[2] = (w2[2] & 0xffffff00) | (wx >> 24); - w2[3] = (w2[3] & 0x000000ff) | (wx << 8); - break; - case 44: w2[3] = wx; - break; - case 45: w2[3] = (w2[3] & 0xff000000) | (wx >> 8); - w3[0] = (w3[0] & 0x00ffffff) | (wx << 24); - break; - case 46: w2[3] = (w2[3] & 0xffff0000) | (wx >> 16); - w3[0] = (w3[0] & 0x0000ffff) | (wx << 16); - break; - case 47: w2[3] = (w2[3] & 0xffffff00) | (wx >> 24); - w3[0] = (w3[0] & 0x000000ff) | (wx << 8); - break; - case 48: w3[0] = wx; - break; - case 49: w3[0] = (w3[0] & 0xff000000) | (wx >> 8); - w3[1] = (w3[1] & 0x00ffffff) | (wx << 24); - break; - case 50: w3[0] = (w3[0] & 0xffff0000) | (wx >> 16); - w3[1] = (w3[1] & 0x0000ffff) | (wx << 16); - break; - case 51: w3[0] = (w3[0] & 0xffffff00) | (wx >> 24); - w3[1] = (w3[1] & 0x000000ff) | (wx << 8); - break; - case 52: w3[1] = wx; - break; - case 53: w3[1] = (w3[1] & 0xff000000) | (wx >> 8); - w3[2] = (w3[2] & 0x00ffffff) | (wx << 24); - break; - case 54: w3[1] = (w3[1] & 0xffff0000) | (wx >> 16); - w3[2] = (w3[2] & 0x0000ffff) | (wx << 16); - break; - case 55: w3[1] = (w3[1] & 0xffffff00) | (wx >> 24); - w3[2] = (w3[2] & 0x000000ff) | (wx << 8); - break; - case 56: w3[2] = wx; - break; - case 57: w3[2] = (w3[2] & 0xff000000) | (wx >> 8); - w3[3] = (w3[3] & 0x00ffffff) | (wx << 24); - break; - case 58: w3[2] = (w3[2] & 0xffff0000) | (wx >> 16); - w3[3] = (w3[3] & 0x0000ffff) | (wx << 16); - break; - case 59: w3[2] = (w3[2] & 0xffffff00) | (wx >> 24); - w3[3] = (w3[3] & 0x000000ff) | (wx << 8); - break; - case 60: w3[3] = wx; - break; - case 61: w3[3] = (w3[3] & 0xff000000) | (wx >> 8); - //w4[0] = (w4[0] & 0x00ffffff) | (wx << 24); - break; - case 62: w3[3] = (w3[3] & 0xffff0000) | (wx >> 16); - //w4[0] = (w4[0] & 0x0000ffff) | (wx << 16); - break; - case 63: w3[3] = (w3[3] & 0xffffff00) | (wx >> 24); - //w4[0] = (w4[0] & 0x000000ff) | (wx << 8); - break; - } -} - -/** - * vector functions as scalar (for outer loop usage) - */ - -inline void append_0x01_1x4_S (u32 w0[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x01; - break; - - case 1: - w0[0] = w0[0] | 0x0100; - break; - - case 2: - w0[0] = w0[0] | 0x010000; - break; - - case 3: - w0[0] = w0[0] | 0x01000000; - break; - - case 4: - w0[1] = 0x01; - break; - - case 5: - w0[1] = w0[1] | 0x0100; - break; - - case 6: - w0[1] = w0[1] | 0x010000; - break; - - case 7: - w0[1] = w0[1] | 0x01000000; - break; - - case 8: - w0[2] = 0x01; - break; - - case 9: - w0[2] = w0[2] | 0x0100; - break; - - case 10: - w0[2] = w0[2] | 0x010000; - break; - - case 11: - w0[2] = w0[2] | 0x01000000; - break; - - case 12: - w0[3] = 0x01; - break; - - case 13: - w0[3] = w0[3] | 0x0100; - break; - - case 14: - w0[3] = w0[3] | 0x010000; - break; - - case 15: - w0[3] = w0[3] | 0x01000000; - break; - } -} - -inline void append_0x01_2x4_S (u32 w0[4], u32 w1[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x01; - break; - - case 1: - w0[0] = w0[0] | 0x0100; - break; - - case 2: - w0[0] = w0[0] | 0x010000; - break; - - case 3: - w0[0] = w0[0] | 0x01000000; - break; - - case 4: - w0[1] = 0x01; - break; - - case 5: - w0[1] = w0[1] | 0x0100; - break; - - case 6: - w0[1] = w0[1] | 0x010000; - break; - - case 7: - w0[1] = w0[1] | 0x01000000; - break; - - case 8: - w0[2] = 0x01; - break; - - case 9: - w0[2] = w0[2] | 0x0100; - break; - - case 10: - w0[2] = w0[2] | 0x010000; - break; - - case 11: - w0[2] = w0[2] | 0x01000000; - break; - - case 12: - w0[3] = 0x01; - break; - - case 13: - w0[3] = w0[3] | 0x0100; - break; - - case 14: - w0[3] = w0[3] | 0x010000; - break; - - case 15: - w0[3] = w0[3] | 0x01000000; - break; - - case 16: - w1[0] = 0x01; - break; - - case 17: - w1[0] = w1[0] | 0x0100; - break; - - case 18: - w1[0] = w1[0] | 0x010000; - break; - - case 19: - w1[0] = w1[0] | 0x01000000; - break; - - case 20: - w1[1] = 0x01; - break; - - case 21: - w1[1] = w1[1] | 0x0100; - break; - - case 22: - w1[1] = w1[1] | 0x010000; - break; - - case 23: - w1[1] = w1[1] | 0x01000000; - break; - - case 24: - w1[2] = 0x01; - break; - - case 25: - w1[2] = w1[2] | 0x0100; - break; - - case 26: - w1[2] = w1[2] | 0x010000; - break; - - case 27: - w1[2] = w1[2] | 0x01000000; - break; - - case 28: - w1[3] = 0x01; - break; - - case 29: - w1[3] = w1[3] | 0x0100; - break; - - case 30: - w1[3] = w1[3] | 0x010000; - break; - - case 31: - w1[3] = w1[3] | 0x01000000; - break; - } -} - -inline void append_0x01_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x01; - break; - - case 1: - w0[0] = w0[0] | 0x0100; - break; - - case 2: - w0[0] = w0[0] | 0x010000; - break; - - case 3: - w0[0] = w0[0] | 0x01000000; - break; - - case 4: - w0[1] = 0x01; - break; - - case 5: - w0[1] = w0[1] | 0x0100; - break; - - case 6: - w0[1] = w0[1] | 0x010000; - break; - - case 7: - w0[1] = w0[1] | 0x01000000; - break; - - case 8: - w0[2] = 0x01; - break; - - case 9: - w0[2] = w0[2] | 0x0100; - break; - - case 10: - w0[2] = w0[2] | 0x010000; - break; - - case 11: - w0[2] = w0[2] | 0x01000000; - break; - - case 12: - w0[3] = 0x01; - break; - - case 13: - w0[3] = w0[3] | 0x0100; - break; - - case 14: - w0[3] = w0[3] | 0x010000; - break; - - case 15: - w0[3] = w0[3] | 0x01000000; - break; - - case 16: - w1[0] = 0x01; - break; - - case 17: - w1[0] = w1[0] | 0x0100; - break; - - case 18: - w1[0] = w1[0] | 0x010000; - break; - - case 19: - w1[0] = w1[0] | 0x01000000; - break; - - case 20: - w1[1] = 0x01; - break; - - case 21: - w1[1] = w1[1] | 0x0100; - break; - - case 22: - w1[1] = w1[1] | 0x010000; - break; - - case 23: - w1[1] = w1[1] | 0x01000000; - break; - - case 24: - w1[2] = 0x01; - break; - - case 25: - w1[2] = w1[2] | 0x0100; - break; - - case 26: - w1[2] = w1[2] | 0x010000; - break; - - case 27: - w1[2] = w1[2] | 0x01000000; - break; - - case 28: - w1[3] = 0x01; - break; - - case 29: - w1[3] = w1[3] | 0x0100; - break; - - case 30: - w1[3] = w1[3] | 0x010000; - break; - - case 31: - w1[3] = w1[3] | 0x01000000; - break; - - case 32: - w2[0] = 0x01; - break; - - case 33: - w2[0] = w2[0] | 0x0100; - break; - - case 34: - w2[0] = w2[0] | 0x010000; - break; - - case 35: - w2[0] = w2[0] | 0x01000000; - break; - - case 36: - w2[1] = 0x01; - break; - - case 37: - w2[1] = w2[1] | 0x0100; - break; - - case 38: - w2[1] = w2[1] | 0x010000; - break; - - case 39: - w2[1] = w2[1] | 0x01000000; - break; - - case 40: - w2[2] = 0x01; - break; - - case 41: - w2[2] = w2[2] | 0x0100; - break; - - case 42: - w2[2] = w2[2] | 0x010000; - break; - - case 43: - w2[2] = w2[2] | 0x01000000; - break; - - case 44: - w2[3] = 0x01; - break; - - case 45: - w2[3] = w2[3] | 0x0100; - break; - - case 46: - w2[3] = w2[3] | 0x010000; - break; - - case 47: - w2[3] = w2[3] | 0x01000000; - break; - } -} - -inline void append_0x01_4x4_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x01; - break; - - case 1: - w0[0] = w0[0] | 0x0100; - break; - - case 2: - w0[0] = w0[0] | 0x010000; - break; - - case 3: - w0[0] = w0[0] | 0x01000000; - break; - - case 4: - w0[1] = 0x01; - break; - - case 5: - w0[1] = w0[1] | 0x0100; - break; - - case 6: - w0[1] = w0[1] | 0x010000; - break; - - case 7: - w0[1] = w0[1] | 0x01000000; - break; - - case 8: - w0[2] = 0x01; - break; - - case 9: - w0[2] = w0[2] | 0x0100; - break; - - case 10: - w0[2] = w0[2] | 0x010000; - break; - - case 11: - w0[2] = w0[2] | 0x01000000; - break; - - case 12: - w0[3] = 0x01; - break; - - case 13: - w0[3] = w0[3] | 0x0100; - break; - - case 14: - w0[3] = w0[3] | 0x010000; - break; - - case 15: - w0[3] = w0[3] | 0x01000000; - break; - - case 16: - w1[0] = 0x01; - break; - - case 17: - w1[0] = w1[0] | 0x0100; - break; - - case 18: - w1[0] = w1[0] | 0x010000; - break; - - case 19: - w1[0] = w1[0] | 0x01000000; - break; - - case 20: - w1[1] = 0x01; - break; - - case 21: - w1[1] = w1[1] | 0x0100; - break; - - case 22: - w1[1] = w1[1] | 0x010000; - break; - - case 23: - w1[1] = w1[1] | 0x01000000; - break; - - case 24: - w1[2] = 0x01; - break; - - case 25: - w1[2] = w1[2] | 0x0100; - break; - - case 26: - w1[2] = w1[2] | 0x010000; - break; - - case 27: - w1[2] = w1[2] | 0x01000000; - break; - - case 28: - w1[3] = 0x01; - break; - - case 29: - w1[3] = w1[3] | 0x0100; - break; - - case 30: - w1[3] = w1[3] | 0x010000; - break; - - case 31: - w1[3] = w1[3] | 0x01000000; - break; - - case 32: - w2[0] = 0x01; - break; - - case 33: - w2[0] = w2[0] | 0x0100; - break; - - case 34: - w2[0] = w2[0] | 0x010000; - break; - - case 35: - w2[0] = w2[0] | 0x01000000; - break; - - case 36: - w2[1] = 0x01; - break; - - case 37: - w2[1] = w2[1] | 0x0100; - break; - - case 38: - w2[1] = w2[1] | 0x010000; - break; - - case 39: - w2[1] = w2[1] | 0x01000000; - break; - - case 40: - w2[2] = 0x01; - break; - - case 41: - w2[2] = w2[2] | 0x0100; - break; - - case 42: - w2[2] = w2[2] | 0x010000; - break; - - case 43: - w2[2] = w2[2] | 0x01000000; - break; - - case 44: - w2[3] = 0x01; - break; - - case 45: - w2[3] = w2[3] | 0x0100; - break; - - case 46: - w2[3] = w2[3] | 0x010000; - break; - - case 47: - w2[3] = w2[3] | 0x01000000; - break; - - case 48: - w3[0] = 0x01; - break; - - case 49: - w3[0] = w3[0] | 0x0100; - break; - - case 50: - w3[0] = w3[0] | 0x010000; - break; - - case 51: - w3[0] = w3[0] | 0x01000000; - break; - - case 52: - w3[1] = 0x01; - break; - - case 53: - w3[1] = w3[1] | 0x0100; - break; - - case 54: - w3[1] = w3[1] | 0x010000; - break; - - case 55: - w3[1] = w3[1] | 0x01000000; - break; - - case 56: - w3[2] = 0x01; - break; - - case 57: - w3[2] = w3[2] | 0x0100; - break; - - case 58: - w3[2] = w3[2] | 0x010000; - break; - - case 59: - w3[2] = w3[2] | 0x01000000; - break; - - case 60: - w3[3] = 0x01; - break; - - case 61: - w3[3] = w3[3] | 0x0100; - break; - - case 62: - w3[3] = w3[3] | 0x010000; - break; - - case 63: - w3[3] = w3[3] | 0x01000000; - break; - } -} - -inline void append_0x02_2x4_S (u32 w0[4], u32 w1[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x02; - break; - - case 1: - w0[0] = w0[0] | 0x0200; - break; - - case 2: - w0[0] = w0[0] | 0x020000; - break; - - case 3: - w0[0] = w0[0] | 0x02000000; - break; - - case 4: - w0[1] = 0x02; - break; - - case 5: - w0[1] = w0[1] | 0x0200; - break; - - case 6: - w0[1] = w0[1] | 0x020000; - break; - - case 7: - w0[1] = w0[1] | 0x02000000; - break; - - case 8: - w0[2] = 0x02; - break; - - case 9: - w0[2] = w0[2] | 0x0200; - break; - - case 10: - w0[2] = w0[2] | 0x020000; - break; - - case 11: - w0[2] = w0[2] | 0x02000000; - break; - - case 12: - w0[3] = 0x02; - break; - - case 13: - w0[3] = w0[3] | 0x0200; - break; - - case 14: - w0[3] = w0[3] | 0x020000; - break; - - case 15: - w0[3] = w0[3] | 0x02000000; - break; - - case 16: - w1[0] = 0x02; - break; - - case 17: - w1[0] = w1[0] | 0x0200; - break; - - case 18: - w1[0] = w1[0] | 0x020000; - break; - - case 19: - w1[0] = w1[0] | 0x02000000; - break; - - case 20: - w1[1] = 0x02; - break; - - case 21: - w1[1] = w1[1] | 0x0200; - break; - - case 22: - w1[1] = w1[1] | 0x020000; - break; - - case 23: - w1[1] = w1[1] | 0x02000000; - break; - - case 24: - w1[2] = 0x02; - break; - - case 25: - w1[2] = w1[2] | 0x0200; - break; - - case 26: - w1[2] = w1[2] | 0x020000; - break; - - case 27: - w1[2] = w1[2] | 0x02000000; - break; - - case 28: - w1[3] = 0x02; - break; - - case 29: - w1[3] = w1[3] | 0x0200; - break; - - case 30: - w1[3] = w1[3] | 0x020000; - break; - - case 31: - w1[3] = w1[3] | 0x02000000; - break; - } -} - -inline void append_0x02_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x02; - break; - - case 1: - w0[0] = w0[0] | 0x0200; - break; - - case 2: - w0[0] = w0[0] | 0x020000; - break; - - case 3: - w0[0] = w0[0] | 0x02000000; - break; - - case 4: - w0[1] = 0x02; - break; - - case 5: - w0[1] = w0[1] | 0x0200; - break; - - case 6: - w0[1] = w0[1] | 0x020000; - break; - - case 7: - w0[1] = w0[1] | 0x02000000; - break; - - case 8: - w0[2] = 0x02; - break; - - case 9: - w0[2] = w0[2] | 0x0200; - break; - - case 10: - w0[2] = w0[2] | 0x020000; - break; - - case 11: - w0[2] = w0[2] | 0x02000000; - break; - - case 12: - w0[3] = 0x02; - break; - - case 13: - w0[3] = w0[3] | 0x0200; - break; - - case 14: - w0[3] = w0[3] | 0x020000; - break; - - case 15: - w0[3] = w0[3] | 0x02000000; - break; - - case 16: - w1[0] = 0x02; - break; - - case 17: - w1[0] = w1[0] | 0x0200; - break; - - case 18: - w1[0] = w1[0] | 0x020000; - break; - - case 19: - w1[0] = w1[0] | 0x02000000; - break; - - case 20: - w1[1] = 0x02; - break; - - case 21: - w1[1] = w1[1] | 0x0200; - break; - - case 22: - w1[1] = w1[1] | 0x020000; - break; - - case 23: - w1[1] = w1[1] | 0x02000000; - break; - - case 24: - w1[2] = 0x02; - break; - - case 25: - w1[2] = w1[2] | 0x0200; - break; - - case 26: - w1[2] = w1[2] | 0x020000; - break; - - case 27: - w1[2] = w1[2] | 0x02000000; - break; - - case 28: - w1[3] = 0x02; - break; - - case 29: - w1[3] = w1[3] | 0x0200; - break; - - case 30: - w1[3] = w1[3] | 0x020000; - break; - - case 31: - w1[3] = w1[3] | 0x02000000; - break; - - case 32: - w2[0] = 0x02; - break; - - case 33: - w2[0] = w2[0] | 0x0200; - break; - - case 34: - w2[0] = w2[0] | 0x020000; - break; - - case 35: - w2[0] = w2[0] | 0x02000000; - break; - - case 36: - w2[1] = 0x02; - break; - - case 37: - w2[1] = w2[1] | 0x0200; - break; - - case 38: - w2[1] = w2[1] | 0x020000; - break; - - case 39: - w2[1] = w2[1] | 0x02000000; - break; - - case 40: - w2[2] = 0x02; - break; - - case 41: - w2[2] = w2[2] | 0x0200; - break; - - case 42: - w2[2] = w2[2] | 0x020000; - break; - - case 43: - w2[2] = w2[2] | 0x02000000; - break; - - case 44: - w2[3] = 0x02; - break; - - case 45: - w2[3] = w2[3] | 0x0200; - break; - - case 46: - w2[3] = w2[3] | 0x020000; - break; - - case 47: - w2[3] = w2[3] | 0x02000000; - break; - } -} - -inline void append_0x80_1x4_S (u32 w0[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x80; - break; - - case 1: - w0[0] = w0[0] | 0x8000; - break; - - case 2: - w0[0] = w0[0] | 0x800000; - break; - - case 3: - w0[0] = w0[0] | 0x80000000; - break; - - case 4: - w0[1] = 0x80; - break; - - case 5: - w0[1] = w0[1] | 0x8000; - break; - - case 6: - w0[1] = w0[1] | 0x800000; - break; - - case 7: - w0[1] = w0[1] | 0x80000000; - break; - - case 8: - w0[2] = 0x80; - break; - - case 9: - w0[2] = w0[2] | 0x8000; - break; - - case 10: - w0[2] = w0[2] | 0x800000; - break; - - case 11: - w0[2] = w0[2] | 0x80000000; - break; - - case 12: - w0[3] = 0x80; - break; - - case 13: - w0[3] = w0[3] | 0x8000; - break; - - case 14: - w0[3] = w0[3] | 0x800000; - break; - - case 15: - w0[3] = w0[3] | 0x80000000; - break; - } -} - -inline void append_0x80_2x4_S (u32 w0[4], u32 w1[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x80; - break; - - case 1: - w0[0] = w0[0] | 0x8000; - break; - - case 2: - w0[0] = w0[0] | 0x800000; - break; - - case 3: - w0[0] = w0[0] | 0x80000000; - break; - - case 4: - w0[1] = 0x80; - break; - - case 5: - w0[1] = w0[1] | 0x8000; - break; - - case 6: - w0[1] = w0[1] | 0x800000; - break; - - case 7: - w0[1] = w0[1] | 0x80000000; - break; - - case 8: - w0[2] = 0x80; - break; - - case 9: - w0[2] = w0[2] | 0x8000; - break; - - case 10: - w0[2] = w0[2] | 0x800000; - break; - - case 11: - w0[2] = w0[2] | 0x80000000; - break; - - case 12: - w0[3] = 0x80; - break; - - case 13: - w0[3] = w0[3] | 0x8000; - break; - - case 14: - w0[3] = w0[3] | 0x800000; - break; - - case 15: - w0[3] = w0[3] | 0x80000000; - break; - - case 16: - w1[0] = 0x80; - break; - - case 17: - w1[0] = w1[0] | 0x8000; - break; - - case 18: - w1[0] = w1[0] | 0x800000; - break; - - case 19: - w1[0] = w1[0] | 0x80000000; - break; - - case 20: - w1[1] = 0x80; - break; - - case 21: - w1[1] = w1[1] | 0x8000; - break; - - case 22: - w1[1] = w1[1] | 0x800000; - break; - - case 23: - w1[1] = w1[1] | 0x80000000; - break; - - case 24: - w1[2] = 0x80; - break; - - case 25: - w1[2] = w1[2] | 0x8000; - break; - - case 26: - w1[2] = w1[2] | 0x800000; - break; - - case 27: - w1[2] = w1[2] | 0x80000000; - break; - - case 28: - w1[3] = 0x80; - break; - - case 29: - w1[3] = w1[3] | 0x8000; - break; - - case 30: - w1[3] = w1[3] | 0x800000; - break; - - case 31: - w1[3] = w1[3] | 0x80000000; - break; - } -} - -inline void append_0x80_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x80; - break; - - case 1: - w0[0] = w0[0] | 0x8000; - break; - - case 2: - w0[0] = w0[0] | 0x800000; - break; - - case 3: - w0[0] = w0[0] | 0x80000000; - break; - - case 4: - w0[1] = 0x80; - break; - - case 5: - w0[1] = w0[1] | 0x8000; - break; - - case 6: - w0[1] = w0[1] | 0x800000; - break; - - case 7: - w0[1] = w0[1] | 0x80000000; - break; - - case 8: - w0[2] = 0x80; - break; - - case 9: - w0[2] = w0[2] | 0x8000; - break; - - case 10: - w0[2] = w0[2] | 0x800000; - break; - - case 11: - w0[2] = w0[2] | 0x80000000; - break; - - case 12: - w0[3] = 0x80; - break; - - case 13: - w0[3] = w0[3] | 0x8000; - break; - - case 14: - w0[3] = w0[3] | 0x800000; - break; - - case 15: - w0[3] = w0[3] | 0x80000000; - break; - - case 16: - w1[0] = 0x80; - break; - - case 17: - w1[0] = w1[0] | 0x8000; - break; - - case 18: - w1[0] = w1[0] | 0x800000; - break; - - case 19: - w1[0] = w1[0] | 0x80000000; - break; - - case 20: - w1[1] = 0x80; - break; - - case 21: - w1[1] = w1[1] | 0x8000; - break; - - case 22: - w1[1] = w1[1] | 0x800000; - break; - - case 23: - w1[1] = w1[1] | 0x80000000; - break; - - case 24: - w1[2] = 0x80; - break; - - case 25: - w1[2] = w1[2] | 0x8000; - break; - - case 26: - w1[2] = w1[2] | 0x800000; - break; - - case 27: - w1[2] = w1[2] | 0x80000000; - break; - - case 28: - w1[3] = 0x80; - break; - - case 29: - w1[3] = w1[3] | 0x8000; - break; - - case 30: - w1[3] = w1[3] | 0x800000; - break; - - case 31: - w1[3] = w1[3] | 0x80000000; - break; - - case 32: - w2[0] = 0x80; - break; - - case 33: - w2[0] = w2[0] | 0x8000; - break; - - case 34: - w2[0] = w2[0] | 0x800000; - break; - - case 35: - w2[0] = w2[0] | 0x80000000; - break; - - case 36: - w2[1] = 0x80; - break; - - case 37: - w2[1] = w2[1] | 0x8000; - break; - - case 38: - w2[1] = w2[1] | 0x800000; - break; - - case 39: - w2[1] = w2[1] | 0x80000000; - break; - - case 40: - w2[2] = 0x80; - break; - - case 41: - w2[2] = w2[2] | 0x8000; - break; - - case 42: - w2[2] = w2[2] | 0x800000; - break; - - case 43: - w2[2] = w2[2] | 0x80000000; - break; - - case 44: - w2[3] = 0x80; - break; - - case 45: - w2[3] = w2[3] | 0x8000; - break; - - case 46: - w2[3] = w2[3] | 0x800000; - break; - - case 47: - w2[3] = w2[3] | 0x80000000; - break; - } -} - -inline void append_0x80_4x4_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset) -{ - switch (offset) - { - case 0: - w0[0] = 0x80; - break; - - case 1: - w0[0] = w0[0] | 0x8000; - break; - - case 2: - w0[0] = w0[0] | 0x800000; - break; - - case 3: - w0[0] = w0[0] | 0x80000000; - break; - - case 4: - w0[1] = 0x80; - break; - - case 5: - w0[1] = w0[1] | 0x8000; - break; - - case 6: - w0[1] = w0[1] | 0x800000; - break; - - case 7: - w0[1] = w0[1] | 0x80000000; - break; - - case 8: - w0[2] = 0x80; - break; - - case 9: - w0[2] = w0[2] | 0x8000; - break; - - case 10: - w0[2] = w0[2] | 0x800000; - break; - - case 11: - w0[2] = w0[2] | 0x80000000; - break; - - case 12: - w0[3] = 0x80; - break; - - case 13: - w0[3] = w0[3] | 0x8000; - break; - - case 14: - w0[3] = w0[3] | 0x800000; - break; - - case 15: - w0[3] = w0[3] | 0x80000000; - break; - - case 16: - w1[0] = 0x80; - break; - - case 17: - w1[0] = w1[0] | 0x8000; - break; - - case 18: - w1[0] = w1[0] | 0x800000; - break; - - case 19: - w1[0] = w1[0] | 0x80000000; - break; - - case 20: - w1[1] = 0x80; - break; - - case 21: - w1[1] = w1[1] | 0x8000; - break; - - case 22: - w1[1] = w1[1] | 0x800000; - break; - - case 23: - w1[1] = w1[1] | 0x80000000; - break; - - case 24: - w1[2] = 0x80; - break; - - case 25: - w1[2] = w1[2] | 0x8000; - break; - - case 26: - w1[2] = w1[2] | 0x800000; - break; - - case 27: - w1[2] = w1[2] | 0x80000000; - break; - - case 28: - w1[3] = 0x80; - break; - - case 29: - w1[3] = w1[3] | 0x8000; - break; - - case 30: - w1[3] = w1[3] | 0x800000; - break; - - case 31: - w1[3] = w1[3] | 0x80000000; - break; - - case 32: - w2[0] = 0x80; - break; - - case 33: - w2[0] = w2[0] | 0x8000; - break; - - case 34: - w2[0] = w2[0] | 0x800000; - break; - - case 35: - w2[0] = w2[0] | 0x80000000; - break; - - case 36: - w2[1] = 0x80; - break; - - case 37: - w2[1] = w2[1] | 0x8000; - break; - - case 38: - w2[1] = w2[1] | 0x800000; - break; - - case 39: - w2[1] = w2[1] | 0x80000000; - break; - - case 40: - w2[2] = 0x80; - break; - - case 41: - w2[2] = w2[2] | 0x8000; - break; - - case 42: - w2[2] = w2[2] | 0x800000; - break; - - case 43: - w2[2] = w2[2] | 0x80000000; - break; - - case 44: - w2[3] = 0x80; - break; - - case 45: - w2[3] = w2[3] | 0x8000; - break; - - case 46: - w2[3] = w2[3] | 0x800000; - break; - - case 47: - w2[3] = w2[3] | 0x80000000; - break; - - case 48: - w3[0] = 0x80; - break; - - case 49: - w3[0] = w3[0] | 0x8000; - break; - - case 50: - w3[0] = w3[0] | 0x800000; - break; - - case 51: - w3[0] = w3[0] | 0x80000000; - break; - - case 52: - w3[1] = 0x80; - break; - - case 53: - w3[1] = w3[1] | 0x8000; - break; - - case 54: - w3[1] = w3[1] | 0x800000; - break; - - case 55: - w3[1] = w3[1] | 0x80000000; - break; - - case 56: - w3[2] = 0x80; - break; - - case 57: - w3[2] = w3[2] | 0x8000; - break; - - case 58: - w3[2] = w3[2] | 0x800000; - break; - - case 59: - w3[2] = w3[2] | 0x80000000; - break; - - case 60: - w3[3] = 0x80; - break; - - case 61: - w3[3] = w3[3] | 0x8000; - break; - - case 62: - w3[3] = w3[3] | 0x800000; - break; - - case 63: - w3[3] = w3[3] | 0x80000000; - break; - } -} - -inline void truncate_block_S (u32 w[4], const u32 len) -{ - switch (len) - { - case 0: w[0] &= 0; - w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 1: w[0] &= 0x000000FF; - w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 2: w[0] &= 0x0000FFFF; - w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 3: w[0] &= 0x00FFFFFF; - w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 4: w[1] &= 0; - w[2] &= 0; - w[3] &= 0; - break; - case 5: w[1] &= 0x000000FF; - w[2] &= 0; - w[3] &= 0; - break; - case 6: w[1] &= 0x0000FFFF; - w[2] &= 0; - w[3] &= 0; - break; - case 7: w[1] &= 0x00FFFFFF; - w[2] &= 0; - w[3] &= 0; - break; - case 8: w[2] &= 0; - w[3] &= 0; - break; - case 9: w[2] &= 0x000000FF; - w[3] &= 0; - break; - case 10: w[2] &= 0x0000FFFF; - w[3] &= 0; - break; - case 11: w[2] &= 0x00FFFFFF; - w[3] &= 0; - break; - case 12: w[3] &= 0; - break; - case 13: w[3] &= 0x000000FF; - break; - case 14: w[3] &= 0x0000FFFF; - break; - case 15: w[3] &= 0x00FFFFFF; - break; - } -} - -inline void make_unicode_S (const u32 in[4], u32 out1[4], u32 out2[4]) -{ - #ifdef IS_NV - out2[3] = __byte_perm_S (in[3], 0, 0x7372); - out2[2] = __byte_perm_S (in[3], 0, 0x7170); - out2[1] = __byte_perm_S (in[2], 0, 0x7372); - out2[0] = __byte_perm_S (in[2], 0, 0x7170); - out1[3] = __byte_perm_S (in[1], 0, 0x7372); - out1[2] = __byte_perm_S (in[1], 0, 0x7170); - out1[1] = __byte_perm_S (in[0], 0, 0x7372); - out1[0] = __byte_perm_S (in[0], 0, 0x7170); - #endif - - #if defined IS_AMD || defined IS_GENERIC - out2[3] = ((in[3] >> 8) & 0x00FF0000) | ((in[3] >> 16) & 0x000000FF); - out2[2] = ((in[3] << 8) & 0x00FF0000) | ((in[3] >> 0) & 0x000000FF); - out2[1] = ((in[2] >> 8) & 0x00FF0000) | ((in[2] >> 16) & 0x000000FF); - out2[0] = ((in[2] << 8) & 0x00FF0000) | ((in[2] >> 0) & 0x000000FF); - out1[3] = ((in[1] >> 8) & 0x00FF0000) | ((in[1] >> 16) & 0x000000FF); - out1[2] = ((in[1] << 8) & 0x00FF0000) | ((in[1] >> 0) & 0x000000FF); - out1[1] = ((in[0] >> 8) & 0x00FF0000) | ((in[0] >> 16) & 0x000000FF); - out1[0] = ((in[0] << 8) & 0x00FF0000) | ((in[0] >> 0) & 0x000000FF); - #endif -} - -inline void undo_unicode_S (const u32 in1[4], const u32 in2[4], u32 out[4]) -{ - #ifdef IS_NV - out[0] = __byte_perm_S (in1[0], in1[1], 0x6420); - out[1] = __byte_perm_S (in1[2], in1[3], 0x6420); - out[2] = __byte_perm_S (in2[0], in2[1], 0x6420); - out[3] = __byte_perm_S (in2[2], in2[3], 0x6420); - #endif - - #if defined IS_AMD || defined IS_GENERIC - out[0] = ((in1[0] & 0x000000ff) >> 0) | ((in1[0] & 0x00ff0000) >> 8) - | ((in1[1] & 0x000000ff) << 16) | ((in1[1] & 0x00ff0000) << 8); - out[1] = ((in1[2] & 0x000000ff) >> 0) | ((in1[2] & 0x00ff0000) >> 8) - | ((in1[3] & 0x000000ff) << 16) | ((in1[3] & 0x00ff0000) << 8); - out[2] = ((in2[0] & 0x000000ff) >> 0) | ((in2[0] & 0x00ff0000) >> 8) - | ((in2[1] & 0x000000ff) << 16) | ((in2[1] & 0x00ff0000) << 8); - out[3] = ((in2[2] & 0x000000ff) >> 0) | ((in2[2] & 0x00ff0000) >> 8) - | ((in2[3] & 0x000000ff) << 16) | ((in2[3] & 0x00ff0000) << 8); - #endif -} - -inline void switch_buffer_by_offset_le_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset) -{ - #if defined IS_AMD || defined IS_GENERIC - const int offset_mod_4 = offset & 3; - - const int offset_minus_4 = 4 - offset; - - switch (offset / 4) - { - case 0: - w3[2] = amd_bytealign_S ( 0, w3[1], offset_minus_4); - w3[1] = amd_bytealign_S (w3[1], w3[0], offset_minus_4); - w3[0] = amd_bytealign_S (w3[0], w2[3], offset_minus_4); - w2[3] = amd_bytealign_S (w2[3], w2[2], offset_minus_4); - w2[2] = amd_bytealign_S (w2[2], w2[1], offset_minus_4); - w2[1] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); - w2[0] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); - w1[3] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); - w1[2] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); - w1[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); - w1[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w0[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w0[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w0[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w0[0] = amd_bytealign_S (w0[0], 0, offset_minus_4); - - if (offset_mod_4 == 0) - { - w0[0] = w0[1]; - w0[1] = w0[2]; - w0[2] = w0[3]; - w0[3] = w1[0]; - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 1: - w3[2] = amd_bytealign_S ( 0, w3[0], offset_minus_4); - w3[1] = amd_bytealign_S (w3[0], w2[3], offset_minus_4); - w3[0] = amd_bytealign_S (w2[3], w2[2], offset_minus_4); - w2[3] = amd_bytealign_S (w2[2], w2[1], offset_minus_4); - w2[2] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); - w2[1] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); - w2[0] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); - w1[3] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); - w1[2] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); - w1[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w1[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w0[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w0[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w0[1] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w0[1] = w0[2]; - w0[2] = w0[3]; - w0[3] = w1[0]; - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 2: - w3[2] = amd_bytealign_S ( 0, w2[3], offset_minus_4); - w3[1] = amd_bytealign_S (w2[3], w2[2], offset_minus_4); - w3[0] = amd_bytealign_S (w2[2], w2[1], offset_minus_4); - w2[3] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); - w2[2] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); - w2[1] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); - w2[0] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); - w1[3] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); - w1[2] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w1[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w1[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w0[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w0[2] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w0[2] = w0[3]; - w0[3] = w1[0]; - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 3: - w3[2] = amd_bytealign_S ( 0, w2[2], offset_minus_4); - w3[1] = amd_bytealign_S (w2[2], w2[1], offset_minus_4); - w3[0] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); - w2[3] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); - w2[2] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); - w2[1] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); - w2[0] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); - w1[3] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w1[2] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w1[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w1[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w0[3] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w0[3] = w1[0]; - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 4: - w3[2] = amd_bytealign_S ( 0, w2[1], offset_minus_4); - w3[1] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); - w3[0] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); - w2[3] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); - w2[2] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); - w2[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); - w2[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w1[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w1[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w1[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w1[0] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w1[0] = w1[1]; - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 5: - w3[2] = amd_bytealign_S ( 0, w2[0], offset_minus_4); - w3[1] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); - w3[0] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); - w2[3] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); - w2[2] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); - w2[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w2[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w1[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w1[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w1[1] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w1[1] = w1[2]; - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 6: - w3[2] = amd_bytealign_S ( 0, w1[3], offset_minus_4); - w3[1] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); - w3[0] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); - w2[3] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); - w2[2] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w2[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w2[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w1[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w1[2] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w1[2] = w1[3]; - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 7: - w3[2] = amd_bytealign_S ( 0, w1[2], offset_minus_4); - w3[1] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); - w3[0] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); - w2[3] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w2[2] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w2[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w2[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w1[3] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w1[3] = w2[0]; - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 8: - w3[2] = amd_bytealign_S ( 0, w1[1], offset_minus_4); - w3[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); - w3[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w2[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w2[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w2[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w2[0] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w2[0] = w2[1]; - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 9: - w3[2] = amd_bytealign_S ( 0, w1[0], offset_minus_4); - w3[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); - w3[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w2[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w2[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w2[1] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w2[1] = w2[2]; - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 10: - w3[2] = amd_bytealign_S ( 0, w0[3], offset_minus_4); - w3[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); - w3[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w2[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w2[2] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w2[2] = w2[3]; - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 11: - w3[2] = amd_bytealign_S ( 0, w0[2], offset_minus_4); - w3[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); - w3[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w2[3] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w2[3] = w3[0]; - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 12: - w3[2] = amd_bytealign_S ( 0, w0[1], offset_minus_4); - w3[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); - w3[0] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w3[0] = w3[1]; - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - - case 13: - w3[2] = amd_bytealign_S ( 0, w0[0], offset_minus_4); - w3[1] = amd_bytealign_S (w0[0], 0, offset_minus_4); - w3[0] = 0; - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - if (offset_mod_4 == 0) - { - w3[1] = w3[2]; - w3[2] = 0; - } - - break; - } - #endif - - #ifdef IS_NV - const int offset_minus_4 = 4 - (offset % 4); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - switch (offset / 4) - { - case 0: - w3[1] = __byte_perm_S (w3[0], w3[1], selector); - w3[0] = __byte_perm_S (w2[3], w3[0], selector); - w2[3] = __byte_perm_S (w2[2], w2[3], selector); - w2[2] = __byte_perm_S (w2[1], w2[2], selector); - w2[1] = __byte_perm_S (w2[0], w2[1], selector); - w2[0] = __byte_perm_S (w1[3], w2[0], selector); - w1[3] = __byte_perm_S (w1[2], w1[3], selector); - w1[2] = __byte_perm_S (w1[1], w1[2], selector); - w1[1] = __byte_perm_S (w1[0], w1[1], selector); - w1[0] = __byte_perm_S (w0[3], w1[0], selector); - w0[3] = __byte_perm_S (w0[2], w0[3], selector); - w0[2] = __byte_perm_S (w0[1], w0[2], selector); - w0[1] = __byte_perm_S (w0[0], w0[1], selector); - w0[0] = __byte_perm_S ( 0, w0[0], selector); - - break; - - case 1: - w3[1] = __byte_perm_S (w2[3], w3[0], selector); - w3[0] = __byte_perm_S (w2[2], w2[3], selector); - w2[3] = __byte_perm_S (w2[1], w2[2], selector); - w2[2] = __byte_perm_S (w2[0], w2[1], selector); - w2[1] = __byte_perm_S (w1[3], w2[0], selector); - w2[0] = __byte_perm_S (w1[2], w1[3], selector); - w1[3] = __byte_perm_S (w1[1], w1[2], selector); - w1[2] = __byte_perm_S (w1[0], w1[1], selector); - w1[1] = __byte_perm_S (w0[3], w1[0], selector); - w1[0] = __byte_perm_S (w0[2], w0[3], selector); - w0[3] = __byte_perm_S (w0[1], w0[2], selector); - w0[2] = __byte_perm_S (w0[0], w0[1], selector); - w0[1] = __byte_perm_S ( 0, w0[0], selector); - w0[0] = 0; - - break; - - case 2: - w3[1] = __byte_perm_S (w2[2], w2[3], selector); - w3[0] = __byte_perm_S (w2[1], w2[2], selector); - w2[3] = __byte_perm_S (w2[0], w2[1], selector); - w2[2] = __byte_perm_S (w1[3], w2[0], selector); - w2[1] = __byte_perm_S (w1[2], w1[3], selector); - w2[0] = __byte_perm_S (w1[1], w1[2], selector); - w1[3] = __byte_perm_S (w1[0], w1[1], selector); - w1[2] = __byte_perm_S (w0[3], w1[0], selector); - w1[1] = __byte_perm_S (w0[2], w0[3], selector); - w1[0] = __byte_perm_S (w0[1], w0[2], selector); - w0[3] = __byte_perm_S (w0[0], w0[1], selector); - w0[2] = __byte_perm_S ( 0, w0[0], selector); - w0[1] = 0; - w0[0] = 0; - - break; - - case 3: - w3[1] = __byte_perm_S (w2[1], w2[2], selector); - w3[0] = __byte_perm_S (w2[0], w2[1], selector); - w2[3] = __byte_perm_S (w1[3], w2[0], selector); - w2[2] = __byte_perm_S (w1[2], w1[3], selector); - w2[1] = __byte_perm_S (w1[1], w1[2], selector); - w2[0] = __byte_perm_S (w1[0], w1[1], selector); - w1[3] = __byte_perm_S (w0[3], w1[0], selector); - w1[2] = __byte_perm_S (w0[2], w0[3], selector); - w1[1] = __byte_perm_S (w0[1], w0[2], selector); - w1[0] = __byte_perm_S (w0[0], w0[1], selector); - w0[3] = __byte_perm_S ( 0, w0[0], selector); - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 4: - w3[1] = __byte_perm_S (w2[0], w2[1], selector); - w3[0] = __byte_perm_S (w1[3], w2[0], selector); - w2[3] = __byte_perm_S (w1[2], w1[3], selector); - w2[2] = __byte_perm_S (w1[1], w1[2], selector); - w2[1] = __byte_perm_S (w1[0], w1[1], selector); - w2[0] = __byte_perm_S (w0[3], w1[0], selector); - w1[3] = __byte_perm_S (w0[2], w0[3], selector); - w1[2] = __byte_perm_S (w0[1], w0[2], selector); - w1[1] = __byte_perm_S (w0[0], w0[1], selector); - w1[0] = __byte_perm_S ( 0, w0[0], selector); - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 5: - w3[1] = __byte_perm_S (w1[3], w2[0], selector); - w3[0] = __byte_perm_S (w1[2], w1[3], selector); - w2[3] = __byte_perm_S (w1[1], w1[2], selector); - w2[2] = __byte_perm_S (w1[0], w1[1], selector); - w2[1] = __byte_perm_S (w0[3], w1[0], selector); - w2[0] = __byte_perm_S (w0[2], w0[3], selector); - w1[3] = __byte_perm_S (w0[1], w0[2], selector); - w1[2] = __byte_perm_S (w0[0], w0[1], selector); - w1[1] = __byte_perm_S ( 0, w0[0], selector); - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 6: - w3[1] = __byte_perm_S (w1[2], w1[3], selector); - w3[0] = __byte_perm_S (w1[1], w1[2], selector); - w2[3] = __byte_perm_S (w1[0], w1[1], selector); - w2[2] = __byte_perm_S (w0[3], w1[0], selector); - w2[1] = __byte_perm_S (w0[2], w0[3], selector); - w2[0] = __byte_perm_S (w0[1], w0[2], selector); - w1[3] = __byte_perm_S (w0[0], w0[1], selector); - w1[2] = __byte_perm_S ( 0, w0[0], selector); - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 7: - w3[1] = __byte_perm_S (w1[1], w1[2], selector); - w3[0] = __byte_perm_S (w1[0], w1[1], selector); - w2[3] = __byte_perm_S (w0[3], w1[0], selector); - w2[2] = __byte_perm_S (w0[2], w0[3], selector); - w2[1] = __byte_perm_S (w0[1], w0[2], selector); - w2[0] = __byte_perm_S (w0[0], w0[1], selector); - w1[3] = __byte_perm_S ( 0, w0[0], selector); - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 8: - w3[1] = __byte_perm_S (w1[0], w1[1], selector); - w3[0] = __byte_perm_S (w0[3], w1[0], selector); - w2[3] = __byte_perm_S (w0[2], w0[3], selector); - w2[2] = __byte_perm_S (w0[1], w0[2], selector); - w2[1] = __byte_perm_S (w0[0], w0[1], selector); - w2[0] = __byte_perm_S ( 0, w0[0], selector); - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 9: - w3[1] = __byte_perm_S (w0[3], w1[0], selector); - w3[0] = __byte_perm_S (w0[2], w0[3], selector); - w2[3] = __byte_perm_S (w0[1], w0[2], selector); - w2[2] = __byte_perm_S (w0[0], w0[1], selector); - w2[1] = __byte_perm_S ( 0, w0[0], selector); - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 10: - w3[1] = __byte_perm_S (w0[2], w0[3], selector); - w3[0] = __byte_perm_S (w0[1], w0[2], selector); - w2[3] = __byte_perm_S (w0[0], w0[1], selector); - w2[2] = __byte_perm_S ( 0, w0[0], selector); - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 11: - w3[1] = __byte_perm_S (w0[1], w0[2], selector); - w3[0] = __byte_perm_S (w0[0], w0[1], selector); - w2[3] = __byte_perm_S ( 0, w0[0], selector); - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 12: - w3[1] = __byte_perm_S (w0[0], w0[1], selector); - w3[0] = __byte_perm_S ( 0, w0[0], selector); - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - - case 13: - w3[1] = __byte_perm_S ( 0, w0[0], selector); - w3[0] = 0; - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - - break; - } - #endif -} - -inline void switch_buffer_by_offset_be_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset) -{ - #if defined IS_AMD || defined IS_GENERIC - switch (offset / 4) - { - case 0: - w3[2] = amd_bytealign_S (w3[1], 0, offset); - w3[1] = amd_bytealign_S (w3[0], w3[1], offset); - w3[0] = amd_bytealign_S (w2[3], w3[0], offset); - w2[3] = amd_bytealign_S (w2[2], w2[3], offset); - w2[2] = amd_bytealign_S (w2[1], w2[2], offset); - w2[1] = amd_bytealign_S (w2[0], w2[1], offset); - w2[0] = amd_bytealign_S (w1[3], w2[0], offset); - w1[3] = amd_bytealign_S (w1[2], w1[3], offset); - w1[2] = amd_bytealign_S (w1[1], w1[2], offset); - w1[1] = amd_bytealign_S (w1[0], w1[1], offset); - w1[0] = amd_bytealign_S (w0[3], w1[0], offset); - w0[3] = amd_bytealign_S (w0[2], w0[3], offset); - w0[2] = amd_bytealign_S (w0[1], w0[2], offset); - w0[1] = amd_bytealign_S (w0[0], w0[1], offset); - w0[0] = amd_bytealign_S ( 0, w0[0], offset); - break; - - case 1: - w3[2] = amd_bytealign_S (w3[0], 0, offset); - w3[1] = amd_bytealign_S (w2[3], w3[0], offset); - w3[0] = amd_bytealign_S (w2[2], w2[3], offset); - w2[3] = amd_bytealign_S (w2[1], w2[2], offset); - w2[2] = amd_bytealign_S (w2[0], w2[1], offset); - w2[1] = amd_bytealign_S (w1[3], w2[0], offset); - w2[0] = amd_bytealign_S (w1[2], w1[3], offset); - w1[3] = amd_bytealign_S (w1[1], w1[2], offset); - w1[2] = amd_bytealign_S (w1[0], w1[1], offset); - w1[1] = amd_bytealign_S (w0[3], w1[0], offset); - w1[0] = amd_bytealign_S (w0[2], w0[3], offset); - w0[3] = amd_bytealign_S (w0[1], w0[2], offset); - w0[2] = amd_bytealign_S (w0[0], w0[1], offset); - w0[1] = amd_bytealign_S ( 0, w0[0], offset); - w0[0] = 0; - break; - - case 2: - w3[2] = amd_bytealign_S (w2[3], 0, offset); - w3[1] = amd_bytealign_S (w2[2], w2[3], offset); - w3[0] = amd_bytealign_S (w2[1], w2[2], offset); - w2[3] = amd_bytealign_S (w2[0], w2[1], offset); - w2[2] = amd_bytealign_S (w1[3], w2[0], offset); - w2[1] = amd_bytealign_S (w1[2], w1[3], offset); - w2[0] = amd_bytealign_S (w1[1], w1[2], offset); - w1[3] = amd_bytealign_S (w1[0], w1[1], offset); - w1[2] = amd_bytealign_S (w0[3], w1[0], offset); - w1[1] = amd_bytealign_S (w0[2], w0[3], offset); - w1[0] = amd_bytealign_S (w0[1], w0[2], offset); - w0[3] = amd_bytealign_S (w0[0], w0[1], offset); - w0[2] = amd_bytealign_S ( 0, w0[0], offset); - w0[1] = 0; - w0[0] = 0; - break; - - case 3: - w3[2] = amd_bytealign_S (w2[2], 0, offset); - w3[1] = amd_bytealign_S (w2[1], w2[2], offset); - w3[0] = amd_bytealign_S (w2[0], w2[1], offset); - w2[3] = amd_bytealign_S (w1[3], w2[0], offset); - w2[2] = amd_bytealign_S (w1[2], w1[3], offset); - w2[1] = amd_bytealign_S (w1[1], w1[2], offset); - w2[0] = amd_bytealign_S (w1[0], w1[1], offset); - w1[3] = amd_bytealign_S (w0[3], w1[0], offset); - w1[2] = amd_bytealign_S (w0[2], w0[3], offset); - w1[1] = amd_bytealign_S (w0[1], w0[2], offset); - w1[0] = amd_bytealign_S (w0[0], w0[1], offset); - w0[3] = amd_bytealign_S ( 0, w0[0], offset); - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 4: - w3[2] = amd_bytealign_S (w2[1], 0, offset); - w3[1] = amd_bytealign_S (w2[0], w2[1], offset); - w3[0] = amd_bytealign_S (w1[3], w2[0], offset); - w2[3] = amd_bytealign_S (w1[2], w1[3], offset); - w2[2] = amd_bytealign_S (w1[1], w1[2], offset); - w2[1] = amd_bytealign_S (w1[0], w1[1], offset); - w2[0] = amd_bytealign_S (w0[3], w1[0], offset); - w1[3] = amd_bytealign_S (w0[2], w0[3], offset); - w1[2] = amd_bytealign_S (w0[1], w0[2], offset); - w1[1] = amd_bytealign_S (w0[0], w0[1], offset); - w1[0] = amd_bytealign_S ( 0, w0[0], offset); - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 5: - w3[2] = amd_bytealign_S (w2[0], 0, offset); - w3[1] = amd_bytealign_S (w1[3], w2[0], offset); - w3[0] = amd_bytealign_S (w1[2], w1[3], offset); - w2[3] = amd_bytealign_S (w1[1], w1[2], offset); - w2[2] = amd_bytealign_S (w1[0], w1[1], offset); - w2[1] = amd_bytealign_S (w0[3], w1[0], offset); - w2[0] = amd_bytealign_S (w0[2], w0[3], offset); - w1[3] = amd_bytealign_S (w0[1], w0[2], offset); - w1[2] = amd_bytealign_S (w0[0], w0[1], offset); - w1[1] = amd_bytealign_S ( 0, w0[0], offset); - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 6: - w3[2] = amd_bytealign_S (w1[3], 0, offset); - w3[1] = amd_bytealign_S (w1[2], w1[3], offset); - w3[0] = amd_bytealign_S (w1[1], w1[2], offset); - w2[3] = amd_bytealign_S (w1[0], w1[1], offset); - w2[2] = amd_bytealign_S (w0[3], w1[0], offset); - w2[1] = amd_bytealign_S (w0[2], w0[3], offset); - w2[0] = amd_bytealign_S (w0[1], w0[2], offset); - w1[3] = amd_bytealign_S (w0[0], w0[1], offset); - w1[2] = amd_bytealign_S ( 0, w0[0], offset); - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 7: - w3[2] = amd_bytealign_S (w1[2], 0, offset); - w3[1] = amd_bytealign_S (w1[1], w1[2], offset); - w3[0] = amd_bytealign_S (w1[0], w1[1], offset); - w2[3] = amd_bytealign_S (w0[3], w1[0], offset); - w2[2] = amd_bytealign_S (w0[2], w0[3], offset); - w2[1] = amd_bytealign_S (w0[1], w0[2], offset); - w2[0] = amd_bytealign_S (w0[0], w0[1], offset); - w1[3] = amd_bytealign_S ( 0, w0[0], offset); - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 8: - w3[2] = amd_bytealign_S (w1[1], 0, offset); - w3[1] = amd_bytealign_S (w1[0], w1[1], offset); - w3[0] = amd_bytealign_S (w0[3], w1[0], offset); - w2[3] = amd_bytealign_S (w0[2], w0[3], offset); - w2[2] = amd_bytealign_S (w0[1], w0[2], offset); - w2[1] = amd_bytealign_S (w0[0], w0[1], offset); - w2[0] = amd_bytealign_S ( 0, w0[0], offset); - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 9: - w3[2] = amd_bytealign_S (w1[0], 0, offset); - w3[1] = amd_bytealign_S (w0[3], w1[0], offset); - w3[0] = amd_bytealign_S (w0[2], w0[3], offset); - w2[3] = amd_bytealign_S (w0[1], w0[2], offset); - w2[2] = amd_bytealign_S (w0[0], w0[1], offset); - w2[1] = amd_bytealign_S ( 0, w0[0], offset); - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 10: - w3[2] = amd_bytealign_S (w0[3], 0, offset); - w3[1] = amd_bytealign_S (w0[2], w0[3], offset); - w3[0] = amd_bytealign_S (w0[1], w0[2], offset); - w2[3] = amd_bytealign_S (w0[0], w0[1], offset); - w2[2] = amd_bytealign_S ( 0, w0[0], offset); - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 11: - w3[2] = amd_bytealign_S (w0[2], 0, offset); - w3[1] = amd_bytealign_S (w0[1], w0[2], offset); - w3[0] = amd_bytealign_S (w0[0], w0[1], offset); - w2[3] = amd_bytealign_S ( 0, w0[0], offset); - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 12: - w3[2] = amd_bytealign_S (w0[1], 0, offset); - w3[1] = amd_bytealign_S (w0[0], w0[1], offset); - w3[0] = amd_bytealign_S ( 0, w0[0], offset); - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 13: - w3[2] = amd_bytealign_S (w0[0], 0, offset); - w3[1] = amd_bytealign_S ( 0, w0[0], offset); - w3[0] = 0; - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - } - #endif - - #ifdef IS_NV - const int selector = (0x76543210 >> ((offset & 3) * 4)) & 0xffff; - - switch (offset / 4) - { - case 0: - w3[1] = __byte_perm_S (w3[1], w3[0], selector); - w3[0] = __byte_perm_S (w3[0], w2[3], selector); - w2[3] = __byte_perm_S (w2[3], w2[2], selector); - w2[2] = __byte_perm_S (w2[2], w2[1], selector); - w2[1] = __byte_perm_S (w2[1], w2[0], selector); - w2[0] = __byte_perm_S (w2[0], w1[3], selector); - w1[3] = __byte_perm_S (w1[3], w1[2], selector); - w1[2] = __byte_perm_S (w1[2], w1[1], selector); - w1[1] = __byte_perm_S (w1[1], w1[0], selector); - w1[0] = __byte_perm_S (w1[0], w0[3], selector); - w0[3] = __byte_perm_S (w0[3], w0[2], selector); - w0[2] = __byte_perm_S (w0[2], w0[1], selector); - w0[1] = __byte_perm_S (w0[1], w0[0], selector); - w0[0] = __byte_perm_S (w0[0], 0, selector); - break; - - case 1: - w3[1] = __byte_perm_S (w3[0], w2[3], selector); - w3[0] = __byte_perm_S (w2[3], w2[2], selector); - w2[3] = __byte_perm_S (w2[2], w2[1], selector); - w2[2] = __byte_perm_S (w2[1], w2[0], selector); - w2[1] = __byte_perm_S (w2[0], w1[3], selector); - w2[0] = __byte_perm_S (w1[3], w1[2], selector); - w1[3] = __byte_perm_S (w1[2], w1[1], selector); - w1[2] = __byte_perm_S (w1[1], w1[0], selector); - w1[1] = __byte_perm_S (w1[0], w0[3], selector); - w1[0] = __byte_perm_S (w0[3], w0[2], selector); - w0[3] = __byte_perm_S (w0[2], w0[1], selector); - w0[2] = __byte_perm_S (w0[1], w0[0], selector); - w0[1] = __byte_perm_S (w0[0], 0, selector); - w0[0] = 0; - break; - - case 2: - w3[1] = __byte_perm_S (w2[3], w2[2], selector); - w3[0] = __byte_perm_S (w2[2], w2[1], selector); - w2[3] = __byte_perm_S (w2[1], w2[0], selector); - w2[2] = __byte_perm_S (w2[0], w1[3], selector); - w2[1] = __byte_perm_S (w1[3], w1[2], selector); - w2[0] = __byte_perm_S (w1[2], w1[1], selector); - w1[3] = __byte_perm_S (w1[1], w1[0], selector); - w1[2] = __byte_perm_S (w1[0], w0[3], selector); - w1[1] = __byte_perm_S (w0[3], w0[2], selector); - w1[0] = __byte_perm_S (w0[2], w0[1], selector); - w0[3] = __byte_perm_S (w0[1], w0[0], selector); - w0[2] = __byte_perm_S (w0[0], 0, selector); - w0[1] = 0; - w0[0] = 0; - break; - - case 3: - w3[1] = __byte_perm_S (w2[2], w2[1], selector); - w3[0] = __byte_perm_S (w2[1], w2[0], selector); - w2[3] = __byte_perm_S (w2[0], w1[3], selector); - w2[2] = __byte_perm_S (w1[3], w1[2], selector); - w2[1] = __byte_perm_S (w1[2], w1[1], selector); - w2[0] = __byte_perm_S (w1[1], w1[0], selector); - w1[3] = __byte_perm_S (w1[0], w0[3], selector); - w1[2] = __byte_perm_S (w0[3], w0[2], selector); - w1[1] = __byte_perm_S (w0[2], w0[1], selector); - w1[0] = __byte_perm_S (w0[1], w0[0], selector); - w0[3] = __byte_perm_S (w0[0], 0, selector); - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 4: - w3[1] = __byte_perm_S (w2[1], w2[0], selector); - w3[0] = __byte_perm_S (w2[0], w1[3], selector); - w2[3] = __byte_perm_S (w1[3], w1[2], selector); - w2[2] = __byte_perm_S (w1[2], w1[1], selector); - w2[1] = __byte_perm_S (w1[1], w1[0], selector); - w2[0] = __byte_perm_S (w1[0], w0[3], selector); - w1[3] = __byte_perm_S (w0[3], w0[2], selector); - w1[2] = __byte_perm_S (w0[2], w0[1], selector); - w1[1] = __byte_perm_S (w0[1], w0[0], selector); - w1[0] = __byte_perm_S (w0[0], 0, selector); - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 5: - w3[1] = __byte_perm_S (w2[0], w1[3], selector); - w3[0] = __byte_perm_S (w1[3], w1[2], selector); - w2[3] = __byte_perm_S (w1[2], w1[1], selector); - w2[2] = __byte_perm_S (w1[1], w1[0], selector); - w2[1] = __byte_perm_S (w1[0], w0[3], selector); - w2[0] = __byte_perm_S (w0[3], w0[2], selector); - w1[3] = __byte_perm_S (w0[2], w0[1], selector); - w1[2] = __byte_perm_S (w0[1], w0[0], selector); - w1[1] = __byte_perm_S (w0[0], 0, selector); - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 6: - w3[1] = __byte_perm_S (w1[3], w1[2], selector); - w3[0] = __byte_perm_S (w1[2], w1[1], selector); - w2[3] = __byte_perm_S (w1[1], w1[0], selector); - w2[2] = __byte_perm_S (w1[0], w0[3], selector); - w2[1] = __byte_perm_S (w0[3], w0[2], selector); - w2[0] = __byte_perm_S (w0[2], w0[1], selector); - w1[3] = __byte_perm_S (w0[1], w0[0], selector); - w1[2] = __byte_perm_S (w0[0], 0, selector); - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 7: - w3[1] = __byte_perm_S (w1[2], w1[1], selector); - w3[0] = __byte_perm_S (w1[1], w1[0], selector); - w2[3] = __byte_perm_S (w1[0], w0[3], selector); - w2[2] = __byte_perm_S (w0[3], w0[2], selector); - w2[1] = __byte_perm_S (w0[2], w0[1], selector); - w2[0] = __byte_perm_S (w0[1], w0[0], selector); - w1[3] = __byte_perm_S (w0[0], 0, selector); - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 8: - w3[1] = __byte_perm_S (w1[1], w1[0], selector); - w3[0] = __byte_perm_S (w1[0], w0[3], selector); - w2[3] = __byte_perm_S (w0[3], w0[2], selector); - w2[2] = __byte_perm_S (w0[2], w0[1], selector); - w2[1] = __byte_perm_S (w0[1], w0[0], selector); - w2[0] = __byte_perm_S (w0[0], 0, selector); - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 9: - w3[1] = __byte_perm_S (w1[0], w0[3], selector); - w3[0] = __byte_perm_S (w0[3], w0[2], selector); - w2[3] = __byte_perm_S (w0[2], w0[1], selector); - w2[2] = __byte_perm_S (w0[1], w0[0], selector); - w2[1] = __byte_perm_S (w0[0], 0, selector); - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 10: - w3[1] = __byte_perm_S (w0[3], w0[2], selector); - w3[0] = __byte_perm_S (w0[2], w0[1], selector); - w2[3] = __byte_perm_S (w0[1], w0[0], selector); - w2[2] = __byte_perm_S (w0[0], 0, selector); - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 11: - w3[1] = __byte_perm_S (w0[2], w0[1], selector); - w3[0] = __byte_perm_S (w0[1], w0[0], selector); - w2[3] = __byte_perm_S (w0[0], 0, selector); - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 12: - w3[1] = __byte_perm_S (w0[1], w0[0], selector); - w3[0] = __byte_perm_S (w0[0], 0, selector); - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - - case 13: - w3[1] = __byte_perm_S (w0[0], 0, selector); - w3[0] = 0; - w2[3] = 0; - w2[2] = 0; - w2[1] = 0; - w2[0] = 0; - w1[3] = 0; - w1[2] = 0; - w1[1] = 0; - w1[0] = 0; - w0[3] = 0; - w0[2] = 0; - w0[1] = 0; - w0[0] = 0; - break; - } - #endif -} - -/** - * vector functions on scalar types (for inner loop usage) - */ - -#define PACKVS2(sn,vn,e) \ - sn[0] = vn[0].s##e; \ - sn[1] = vn[1].s##e; - -#define PACKSV2(sn,vn,e) \ - vn[0].s##e = sn[0]; \ - vn[1].s##e = sn[1]; - -#define PACKVS24(s0,s1,v0,v1,e) \ - PACKVS4 (s0, v0, e); \ - PACKVS4 (s1, v1, e); - -#define PACKSV24(s0,s1,v0,v1,e) \ - PACKSV4 (s0, v0, e); \ - PACKSV4 (s1, v1, e); - -#define PACKVS4(sn,vn,e) \ - sn[0] = vn[0].s##e; \ - sn[1] = vn[1].s##e; \ - sn[2] = vn[2].s##e; \ - sn[3] = vn[3].s##e; - -#define PACKSV4(sn,vn,e) \ - vn[0].s##e = sn[0]; \ - vn[1].s##e = sn[1]; \ - vn[2].s##e = sn[2]; \ - vn[3].s##e = sn[3]; - -#define PACKVS44(s0,s1,s2,s3,v0,v1,v2,v3,e) \ - PACKVS4 (s0, v0, e); \ - PACKVS4 (s1, v1, e); \ - PACKVS4 (s2, v2, e); \ - PACKVS4 (s3, v3, e); - -#define PACKSV44(s0,s1,s2,s3,v0,v1,v2,v3,e) \ - PACKSV4 (s0, v0, e); \ - PACKSV4 (s1, v1, e); \ - PACKSV4 (s2, v2, e); \ - PACKSV4 (s3, v3, e); - -inline void switch_buffer_by_offset_le_VV (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x offset) -{ - #if VECT_SIZE == 1 - - switch_buffer_by_offset_le_S (w0, w1, w2, w3, offset); - - #else - - u32 t0[4]; - u32 t1[4]; - u32 t2[4]; - u32 t3[4]; - - #endif - - #if VECT_SIZE == 2 - - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); - - #elif VECT_SIZE == 4 - - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); - - #elif VECT_SIZE == 8 - - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); - - #elif VECT_SIZE == 16 - - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s8); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s9); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, a); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sa); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, a); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, b); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sb); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, b); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, c); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sc); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, c); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, d); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sd); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, d); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, e); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.se); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, e); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, f); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sf); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, f); - - #endif -} - -inline void append_0x01_2x4_VV (u32x w0[4], u32x w1[4], const u32x offset) -{ - #if VECT_SIZE == 1 - - append_0x01_2x4_S (w0, w1, offset); - - #else - - u32 t0[4]; - u32 t1[4]; - - #endif - - #if VECT_SIZE == 2 - - PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); - PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); - - #elif VECT_SIZE == 4 - - PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); - PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); - PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); - PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); - - #elif VECT_SIZE == 8 - - PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); - PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); - PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); - PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); - PACKVS24 (t0, t1, w0, w1, 4); append_0x01_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4); - PACKVS24 (t0, t1, w0, w1, 5); append_0x01_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5); - PACKVS24 (t0, t1, w0, w1, 6); append_0x01_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6); - PACKVS24 (t0, t1, w0, w1, 7); append_0x01_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7); - - #elif VECT_SIZE == 16 - - PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); - PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); - PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); - PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); - PACKVS24 (t0, t1, w0, w1, 4); append_0x01_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4); - PACKVS24 (t0, t1, w0, w1, 5); append_0x01_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5); - PACKVS24 (t0, t1, w0, w1, 6); append_0x01_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6); - PACKVS24 (t0, t1, w0, w1, 7); append_0x01_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7); - PACKVS24 (t0, t1, w0, w1, 8); append_0x01_2x4_S (t0, t1, offset.s8); PACKSV24 (t0, t1, w0, w1, 8); - PACKVS24 (t0, t1, w0, w1, 9); append_0x01_2x4_S (t0, t1, offset.s9); PACKSV24 (t0, t1, w0, w1, 9); - PACKVS24 (t0, t1, w0, w1, a); append_0x01_2x4_S (t0, t1, offset.sa); PACKSV24 (t0, t1, w0, w1, a); - PACKVS24 (t0, t1, w0, w1, b); append_0x01_2x4_S (t0, t1, offset.sb); PACKSV24 (t0, t1, w0, w1, b); - PACKVS24 (t0, t1, w0, w1, c); append_0x01_2x4_S (t0, t1, offset.sc); PACKSV24 (t0, t1, w0, w1, c); - PACKVS24 (t0, t1, w0, w1, d); append_0x01_2x4_S (t0, t1, offset.sd); PACKSV24 (t0, t1, w0, w1, d); - PACKVS24 (t0, t1, w0, w1, e); append_0x01_2x4_S (t0, t1, offset.se); PACKSV24 (t0, t1, w0, w1, e); - PACKVS24 (t0, t1, w0, w1, f); append_0x01_2x4_S (t0, t1, offset.sf); PACKSV24 (t0, t1, w0, w1, f); - - #endif -} - -inline void append_0x80_2x4_VV (u32x w0[4], u32x w1[4], const u32x offset) -{ - #if VECT_SIZE == 1 - - append_0x80_2x4_S (w0, w1, offset); - - #else - - u32 t0[4]; - u32 t1[4]; - - #endif - - #if VECT_SIZE == 2 - - PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); - PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); - - #elif VECT_SIZE == 4 - - PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); - PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); - PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); - PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); - - #elif VECT_SIZE == 8 - - PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); - PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); - PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); - PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); - PACKVS24 (t0, t1, w0, w1, 4); append_0x80_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4); - PACKVS24 (t0, t1, w0, w1, 5); append_0x80_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5); - PACKVS24 (t0, t1, w0, w1, 6); append_0x80_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6); - PACKVS24 (t0, t1, w0, w1, 7); append_0x80_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7); - - #elif VECT_SIZE == 16 - - PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); - PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); - PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); - PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); - PACKVS24 (t0, t1, w0, w1, 4); append_0x80_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4); - PACKVS24 (t0, t1, w0, w1, 5); append_0x80_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5); - PACKVS24 (t0, t1, w0, w1, 6); append_0x80_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6); - PACKVS24 (t0, t1, w0, w1, 7); append_0x80_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7); - PACKVS24 (t0, t1, w0, w1, 8); append_0x80_2x4_S (t0, t1, offset.s8); PACKSV24 (t0, t1, w0, w1, 8); - PACKVS24 (t0, t1, w0, w1, 9); append_0x80_2x4_S (t0, t1, offset.s9); PACKSV24 (t0, t1, w0, w1, 9); - PACKVS24 (t0, t1, w0, w1, a); append_0x80_2x4_S (t0, t1, offset.sa); PACKSV24 (t0, t1, w0, w1, a); - PACKVS24 (t0, t1, w0, w1, b); append_0x80_2x4_S (t0, t1, offset.sb); PACKSV24 (t0, t1, w0, w1, b); - PACKVS24 (t0, t1, w0, w1, c); append_0x80_2x4_S (t0, t1, offset.sc); PACKSV24 (t0, t1, w0, w1, c); - PACKVS24 (t0, t1, w0, w1, d); append_0x80_2x4_S (t0, t1, offset.sd); PACKSV24 (t0, t1, w0, w1, d); - PACKVS24 (t0, t1, w0, w1, e); append_0x80_2x4_S (t0, t1, offset.se); PACKSV24 (t0, t1, w0, w1, e); - PACKVS24 (t0, t1, w0, w1, f); append_0x80_2x4_S (t0, t1, offset.sf); PACKSV24 (t0, t1, w0, w1, f); - - #endif -} - -inline void append_0x80_4x4_VV (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x offset) -{ - #if VECT_SIZE == 1 - - append_0x80_4x4_S (w0, w1, w2, w3, offset); - - #else - - u32 t0[4]; - u32 t1[4]; - u32 t2[4]; - u32 t3[4]; - - #endif - - #if VECT_SIZE == 2 - - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); - - #elif VECT_SIZE == 4 - - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); - - #elif VECT_SIZE == 8 - - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); append_0x80_4x4_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); append_0x80_4x4_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); append_0x80_4x4_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); append_0x80_4x4_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); - - #elif VECT_SIZE == 16 - - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); append_0x80_4x4_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); append_0x80_4x4_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); append_0x80_4x4_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); append_0x80_4x4_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); append_0x80_4x4_S (t0, t1, t2, t3, offset.s8); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); append_0x80_4x4_S (t0, t1, t2, t3, offset.s9); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, a); append_0x80_4x4_S (t0, t1, t2, t3, offset.sa); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, a); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, b); append_0x80_4x4_S (t0, t1, t2, t3, offset.sb); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, b); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, c); append_0x80_4x4_S (t0, t1, t2, t3, offset.sc); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, c); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, d); append_0x80_4x4_S (t0, t1, t2, t3, offset.sd); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, d); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, e); append_0x80_4x4_S (t0, t1, t2, t3, offset.se); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, e); - PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, f); append_0x80_4x4_S (t0, t1, t2, t3, offset.sf); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, f); - - #endif -} diff --git a/OpenCL/inc_cipher_aes256.cl b/OpenCL/inc_cipher_aes256.cl new file mode 100644 index 0000000..b3b6107 --- /dev/null +++ b/OpenCL/inc_cipher_aes256.cl @@ -0,0 +1,1048 @@ +/** + * Author......: Jens Steube + * License.....: MIT + */ + +__constant u32 te0[256] = +{ + 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, + 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, + 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, + 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, + 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, + 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, + 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, + 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, + 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, + 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, + 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, + 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, + 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, + 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, + 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, + 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, + 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, + 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, + 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, + 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, + 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, + 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, + 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, + 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, + 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, + 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, + 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, + 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, + 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, + 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, + 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, + 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d, + 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, + 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, + 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, + 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, + 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, + 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, + 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, + 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, + 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, + 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, + 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, + 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b, + 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, + 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, + 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, + 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, + 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, + 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, + 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, + 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, + 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, + 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, + 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, + 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, + 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, + 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, + 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, + 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, + 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, + 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, + 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, + 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, +}; + +__constant u32 te1[256] = +{ + 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, + 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, + 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, + 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, + 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, + 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, + 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, + 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, + 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, + 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, + 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, + 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, + 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, + 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, + 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, + 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, + 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, + 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, + 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, + 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, + 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, + 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, + 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, + 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, + 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, + 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, + 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, + 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, + 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, + 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, + 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, + 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, + 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, + 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, + 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, + 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, + 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, + 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, + 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, + 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, + 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, + 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, + 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, + 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, + 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, + 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, + 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, + 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, + 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, + 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, + 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, + 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, + 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, + 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, + 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, + 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, + 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, + 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, + 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, + 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, + 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, + 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, + 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, + 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, +}; + +__constant u32 te2[256] = +{ + 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, + 0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5, + 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, + 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, + 0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d, + 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, + 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, + 0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0, + 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, + 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, + 0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1, + 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, + 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, + 0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a, + 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, + 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, + 0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a, + 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, + 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, + 0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384, + 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, + 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, + 0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239, + 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, + 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, + 0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185, + 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, + 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, + 0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f, + 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, + 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, + 0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2, + 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, + 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, + 0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d, + 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, + 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, + 0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88, + 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, + 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, + 0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a, + 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, + 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, + 0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279, + 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, + 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, + 0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea, + 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, + 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, + 0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6, + 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, + 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, + 0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66, + 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, + 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, + 0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e, + 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, + 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, + 0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9, + 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, + 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, + 0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068, + 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, + 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, +}; + +__constant u32 te3[256] = +{ + 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, + 0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491, + 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, + 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, + 0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa, + 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, + 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, + 0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b, + 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, + 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, + 0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9, + 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, + 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, + 0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f, + 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, + 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, + 0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34, + 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, + 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, + 0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713, + 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, + 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, + 0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72, + 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, + 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, + 0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411, + 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, + 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, + 0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05, + 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, + 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, + 0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf, + 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, + 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, + 0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a, + 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, + 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, + 0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b, + 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, + 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, + 0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14, + 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, + 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, + 0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2, + 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, + 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, + 0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf, + 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, + 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, + 0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197, + 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, + 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, + 0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc, + 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, + 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, + 0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927, + 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, + 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, + 0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9, + 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, + 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, + 0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0, + 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, + 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, +}; + +__constant u32 te4[256] = +{ + 0x63636363, 0x7c7c7c7c, 0x77777777, 0x7b7b7b7b, + 0xf2f2f2f2, 0x6b6b6b6b, 0x6f6f6f6f, 0xc5c5c5c5, + 0x30303030, 0x01010101, 0x67676767, 0x2b2b2b2b, + 0xfefefefe, 0xd7d7d7d7, 0xabababab, 0x76767676, + 0xcacacaca, 0x82828282, 0xc9c9c9c9, 0x7d7d7d7d, + 0xfafafafa, 0x59595959, 0x47474747, 0xf0f0f0f0, + 0xadadadad, 0xd4d4d4d4, 0xa2a2a2a2, 0xafafafaf, + 0x9c9c9c9c, 0xa4a4a4a4, 0x72727272, 0xc0c0c0c0, + 0xb7b7b7b7, 0xfdfdfdfd, 0x93939393, 0x26262626, + 0x36363636, 0x3f3f3f3f, 0xf7f7f7f7, 0xcccccccc, + 0x34343434, 0xa5a5a5a5, 0xe5e5e5e5, 0xf1f1f1f1, + 0x71717171, 0xd8d8d8d8, 0x31313131, 0x15151515, + 0x04040404, 0xc7c7c7c7, 0x23232323, 0xc3c3c3c3, + 0x18181818, 0x96969696, 0x05050505, 0x9a9a9a9a, + 0x07070707, 0x12121212, 0x80808080, 0xe2e2e2e2, + 0xebebebeb, 0x27272727, 0xb2b2b2b2, 0x75757575, + 0x09090909, 0x83838383, 0x2c2c2c2c, 0x1a1a1a1a, + 0x1b1b1b1b, 0x6e6e6e6e, 0x5a5a5a5a, 0xa0a0a0a0, + 0x52525252, 0x3b3b3b3b, 0xd6d6d6d6, 0xb3b3b3b3, + 0x29292929, 0xe3e3e3e3, 0x2f2f2f2f, 0x84848484, + 0x53535353, 0xd1d1d1d1, 0x00000000, 0xedededed, + 0x20202020, 0xfcfcfcfc, 0xb1b1b1b1, 0x5b5b5b5b, + 0x6a6a6a6a, 0xcbcbcbcb, 0xbebebebe, 0x39393939, + 0x4a4a4a4a, 0x4c4c4c4c, 0x58585858, 0xcfcfcfcf, + 0xd0d0d0d0, 0xefefefef, 0xaaaaaaaa, 0xfbfbfbfb, + 0x43434343, 0x4d4d4d4d, 0x33333333, 0x85858585, + 0x45454545, 0xf9f9f9f9, 0x02020202, 0x7f7f7f7f, + 0x50505050, 0x3c3c3c3c, 0x9f9f9f9f, 0xa8a8a8a8, + 0x51515151, 0xa3a3a3a3, 0x40404040, 0x8f8f8f8f, + 0x92929292, 0x9d9d9d9d, 0x38383838, 0xf5f5f5f5, + 0xbcbcbcbc, 0xb6b6b6b6, 0xdadadada, 0x21212121, + 0x10101010, 0xffffffff, 0xf3f3f3f3, 0xd2d2d2d2, + 0xcdcdcdcd, 0x0c0c0c0c, 0x13131313, 0xecececec, + 0x5f5f5f5f, 0x97979797, 0x44444444, 0x17171717, + 0xc4c4c4c4, 0xa7a7a7a7, 0x7e7e7e7e, 0x3d3d3d3d, + 0x64646464, 0x5d5d5d5d, 0x19191919, 0x73737373, + 0x60606060, 0x81818181, 0x4f4f4f4f, 0xdcdcdcdc, + 0x22222222, 0x2a2a2a2a, 0x90909090, 0x88888888, + 0x46464646, 0xeeeeeeee, 0xb8b8b8b8, 0x14141414, + 0xdededede, 0x5e5e5e5e, 0x0b0b0b0b, 0xdbdbdbdb, + 0xe0e0e0e0, 0x32323232, 0x3a3a3a3a, 0x0a0a0a0a, + 0x49494949, 0x06060606, 0x24242424, 0x5c5c5c5c, + 0xc2c2c2c2, 0xd3d3d3d3, 0xacacacac, 0x62626262, + 0x91919191, 0x95959595, 0xe4e4e4e4, 0x79797979, + 0xe7e7e7e7, 0xc8c8c8c8, 0x37373737, 0x6d6d6d6d, + 0x8d8d8d8d, 0xd5d5d5d5, 0x4e4e4e4e, 0xa9a9a9a9, + 0x6c6c6c6c, 0x56565656, 0xf4f4f4f4, 0xeaeaeaea, + 0x65656565, 0x7a7a7a7a, 0xaeaeaeae, 0x08080808, + 0xbabababa, 0x78787878, 0x25252525, 0x2e2e2e2e, + 0x1c1c1c1c, 0xa6a6a6a6, 0xb4b4b4b4, 0xc6c6c6c6, + 0xe8e8e8e8, 0xdddddddd, 0x74747474, 0x1f1f1f1f, + 0x4b4b4b4b, 0xbdbdbdbd, 0x8b8b8b8b, 0x8a8a8a8a, + 0x70707070, 0x3e3e3e3e, 0xb5b5b5b5, 0x66666666, + 0x48484848, 0x03030303, 0xf6f6f6f6, 0x0e0e0e0e, + 0x61616161, 0x35353535, 0x57575757, 0xb9b9b9b9, + 0x86868686, 0xc1c1c1c1, 0x1d1d1d1d, 0x9e9e9e9e, + 0xe1e1e1e1, 0xf8f8f8f8, 0x98989898, 0x11111111, + 0x69696969, 0xd9d9d9d9, 0x8e8e8e8e, 0x94949494, + 0x9b9b9b9b, 0x1e1e1e1e, 0x87878787, 0xe9e9e9e9, + 0xcececece, 0x55555555, 0x28282828, 0xdfdfdfdf, + 0x8c8c8c8c, 0xa1a1a1a1, 0x89898989, 0x0d0d0d0d, + 0xbfbfbfbf, 0xe6e6e6e6, 0x42424242, 0x68686868, + 0x41414141, 0x99999999, 0x2d2d2d2d, 0x0f0f0f0f, + 0xb0b0b0b0, 0x54545454, 0xbbbbbbbb, 0x16161616, +}; + +__constant u32 td0[256] = +{ + 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, + 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, + 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, + 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, + 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, + 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, + 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, + 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, + 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, + 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, + 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, + 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94, + 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, + 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, + 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, + 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c, + 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, + 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, + 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, + 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, + 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, + 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, + 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, + 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, + 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, + 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, + 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, + 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a, + 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, + 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, + 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, + 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8, + 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, + 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, + 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, + 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, + 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, + 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, + 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, + 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, + 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, + 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, + 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, + 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5, + 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, + 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, + 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, + 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6, + 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, + 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, + 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, + 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, + 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, + 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, + 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, + 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, + 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, + 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, + 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, + 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86, + 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, + 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, + 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, + 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742, +}; + +__constant u32 td1[256] = +{ + 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, + 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, + 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, + 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, + 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, + 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, + 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, + 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, + 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, + 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, + 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, + 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, + 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, + 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, + 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, + 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, + 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, + 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, + 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, + 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, + 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, + 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, + 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, + 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, + 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, + 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, + 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, + 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, + 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, + 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, + 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, + 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, + 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, + 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, + 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, + 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, + 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, + 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, + 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, + 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, + 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, + 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, + 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, + 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, + 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, + 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, + 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, + 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, + 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, + 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, + 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, + 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, + 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, + 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, + 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, + 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, + 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, + 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, + 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, + 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, + 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, + 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, + 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, + 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, +}; + +__constant u32 td2[256] = +{ + 0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27, + 0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3, + 0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502, + 0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562, + 0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe, + 0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3, + 0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552, + 0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9, + 0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9, + 0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce, + 0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253, + 0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908, + 0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b, + 0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655, + 0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337, + 0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16, + 0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69, + 0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6, + 0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6, + 0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e, + 0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6, + 0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050, + 0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9, + 0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8, + 0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000, + 0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a, + 0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d, + 0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436, + 0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b, + 0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12, + 0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b, + 0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e, + 0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f, + 0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb, + 0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4, + 0xdccad731, 0x85104263, 0x22401397, 0x112084c6, + 0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729, + 0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1, + 0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9, + 0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233, + 0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4, + 0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad, + 0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e, + 0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3, + 0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25, + 0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b, + 0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f, + 0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15, + 0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0, + 0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2, + 0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7, + 0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791, + 0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496, + 0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665, + 0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b, + 0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6, + 0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13, + 0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47, + 0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7, + 0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844, + 0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3, + 0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d, + 0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456, + 0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8, +}; + +__constant u32 td3[256] = +{ + 0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a, + 0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b, + 0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5, + 0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5, + 0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d, + 0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b, + 0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95, + 0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e, + 0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27, + 0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d, + 0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562, + 0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9, + 0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752, + 0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66, + 0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3, + 0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced, + 0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e, + 0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4, + 0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4, + 0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd, + 0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d, + 0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60, + 0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767, + 0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79, + 0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000, + 0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c, + 0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736, + 0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24, + 0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b, + 0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c, + 0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12, + 0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814, + 0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3, + 0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b, + 0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8, + 0x31dccad7, 0x63851042, 0x97224013, 0xc6112084, + 0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7, + 0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077, + 0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247, + 0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22, + 0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698, + 0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f, + 0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254, + 0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582, + 0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf, + 0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb, + 0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883, + 0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef, + 0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629, + 0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035, + 0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533, + 0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17, + 0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4, + 0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46, + 0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb, + 0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d, + 0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb, + 0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a, + 0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73, + 0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678, + 0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2, + 0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff, + 0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064, + 0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0, +}; + +__constant u32 td4[256] = +{ + 0x52525252, 0x09090909, 0x6a6a6a6a, 0xd5d5d5d5, + 0x30303030, 0x36363636, 0xa5a5a5a5, 0x38383838, + 0xbfbfbfbf, 0x40404040, 0xa3a3a3a3, 0x9e9e9e9e, + 0x81818181, 0xf3f3f3f3, 0xd7d7d7d7, 0xfbfbfbfb, + 0x7c7c7c7c, 0xe3e3e3e3, 0x39393939, 0x82828282, + 0x9b9b9b9b, 0x2f2f2f2f, 0xffffffff, 0x87878787, + 0x34343434, 0x8e8e8e8e, 0x43434343, 0x44444444, + 0xc4c4c4c4, 0xdededede, 0xe9e9e9e9, 0xcbcbcbcb, + 0x54545454, 0x7b7b7b7b, 0x94949494, 0x32323232, + 0xa6a6a6a6, 0xc2c2c2c2, 0x23232323, 0x3d3d3d3d, + 0xeeeeeeee, 0x4c4c4c4c, 0x95959595, 0x0b0b0b0b, + 0x42424242, 0xfafafafa, 0xc3c3c3c3, 0x4e4e4e4e, + 0x08080808, 0x2e2e2e2e, 0xa1a1a1a1, 0x66666666, + 0x28282828, 0xd9d9d9d9, 0x24242424, 0xb2b2b2b2, + 0x76767676, 0x5b5b5b5b, 0xa2a2a2a2, 0x49494949, + 0x6d6d6d6d, 0x8b8b8b8b, 0xd1d1d1d1, 0x25252525, + 0x72727272, 0xf8f8f8f8, 0xf6f6f6f6, 0x64646464, + 0x86868686, 0x68686868, 0x98989898, 0x16161616, + 0xd4d4d4d4, 0xa4a4a4a4, 0x5c5c5c5c, 0xcccccccc, + 0x5d5d5d5d, 0x65656565, 0xb6b6b6b6, 0x92929292, + 0x6c6c6c6c, 0x70707070, 0x48484848, 0x50505050, + 0xfdfdfdfd, 0xedededed, 0xb9b9b9b9, 0xdadadada, + 0x5e5e5e5e, 0x15151515, 0x46464646, 0x57575757, + 0xa7a7a7a7, 0x8d8d8d8d, 0x9d9d9d9d, 0x84848484, + 0x90909090, 0xd8d8d8d8, 0xabababab, 0x00000000, + 0x8c8c8c8c, 0xbcbcbcbc, 0xd3d3d3d3, 0x0a0a0a0a, + 0xf7f7f7f7, 0xe4e4e4e4, 0x58585858, 0x05050505, + 0xb8b8b8b8, 0xb3b3b3b3, 0x45454545, 0x06060606, + 0xd0d0d0d0, 0x2c2c2c2c, 0x1e1e1e1e, 0x8f8f8f8f, + 0xcacacaca, 0x3f3f3f3f, 0x0f0f0f0f, 0x02020202, + 0xc1c1c1c1, 0xafafafaf, 0xbdbdbdbd, 0x03030303, + 0x01010101, 0x13131313, 0x8a8a8a8a, 0x6b6b6b6b, + 0x3a3a3a3a, 0x91919191, 0x11111111, 0x41414141, + 0x4f4f4f4f, 0x67676767, 0xdcdcdcdc, 0xeaeaeaea, + 0x97979797, 0xf2f2f2f2, 0xcfcfcfcf, 0xcececece, + 0xf0f0f0f0, 0xb4b4b4b4, 0xe6e6e6e6, 0x73737373, + 0x96969696, 0xacacacac, 0x74747474, 0x22222222, + 0xe7e7e7e7, 0xadadadad, 0x35353535, 0x85858585, + 0xe2e2e2e2, 0xf9f9f9f9, 0x37373737, 0xe8e8e8e8, + 0x1c1c1c1c, 0x75757575, 0xdfdfdfdf, 0x6e6e6e6e, + 0x47474747, 0xf1f1f1f1, 0x1a1a1a1a, 0x71717171, + 0x1d1d1d1d, 0x29292929, 0xc5c5c5c5, 0x89898989, + 0x6f6f6f6f, 0xb7b7b7b7, 0x62626262, 0x0e0e0e0e, + 0xaaaaaaaa, 0x18181818, 0xbebebebe, 0x1b1b1b1b, + 0xfcfcfcfc, 0x56565656, 0x3e3e3e3e, 0x4b4b4b4b, + 0xc6c6c6c6, 0xd2d2d2d2, 0x79797979, 0x20202020, + 0x9a9a9a9a, 0xdbdbdbdb, 0xc0c0c0c0, 0xfefefefe, + 0x78787878, 0xcdcdcdcd, 0x5a5a5a5a, 0xf4f4f4f4, + 0x1f1f1f1f, 0xdddddddd, 0xa8a8a8a8, 0x33333333, + 0x88888888, 0x07070707, 0xc7c7c7c7, 0x31313131, + 0xb1b1b1b1, 0x12121212, 0x10101010, 0x59595959, + 0x27272727, 0x80808080, 0xecececec, 0x5f5f5f5f, + 0x60606060, 0x51515151, 0x7f7f7f7f, 0xa9a9a9a9, + 0x19191919, 0xb5b5b5b5, 0x4a4a4a4a, 0x0d0d0d0d, + 0x2d2d2d2d, 0xe5e5e5e5, 0x7a7a7a7a, 0x9f9f9f9f, + 0x93939393, 0xc9c9c9c9, 0x9c9c9c9c, 0xefefefef, + 0xa0a0a0a0, 0xe0e0e0e0, 0x3b3b3b3b, 0x4d4d4d4d, + 0xaeaeaeae, 0x2a2a2a2a, 0xf5f5f5f5, 0xb0b0b0b0, + 0xc8c8c8c8, 0xebebebeb, 0xbbbbbbbb, 0x3c3c3c3c, + 0x83838383, 0x53535353, 0x99999999, 0x61616161, + 0x17171717, 0x2b2b2b2b, 0x04040404, 0x7e7e7e7e, + 0xbabababa, 0x77777777, 0xd6d6d6d6, 0x26262626, + 0xe1e1e1e1, 0x69696969, 0x14141414, 0x63636363, + 0x55555555, 0x21212121, 0x0c0c0c0c, 0x7d7d7d7d, +}; + +__constant u32 rcon[] = +{ + 0x01000000, 0x02000000, 0x04000000, 0x08000000, + 0x10000000, 0x20000000, 0x40000000, 0x80000000, + 0x1b000000, 0x36000000, +}; + +void aes256_ExpandKey (u32 *ks, const u32 *ukey) +{ + ks[0] = ukey[0]; + ks[1] = ukey[1]; + ks[2] = ukey[2]; + ks[3] = ukey[3]; + ks[4] = ukey[4]; + ks[5] = ukey[5]; + ks[6] = ukey[6]; + ks[7] = ukey[7]; + + int i; + int j; + + i = 0; + j = 0; + + while (1) + { + u32 temp = ks[j + 7]; + + ks[j + 8] = ks[j + 0] + ^ (te2[(temp >> 16) & 0xff] & 0xff000000) + ^ (te3[(temp >> 8) & 0xff] & 0x00ff0000) + ^ (te0[(temp >> 0) & 0xff] & 0x0000ff00) + ^ (te1[(temp >> 24) & 0xff] & 0x000000ff) + ^ rcon[i]; + + ks[j + 9] = ks[j + 1] ^ ks[j + 8]; + ks[j + 10] = ks[j + 2] ^ ks[j + 9]; + ks[j + 11] = ks[j + 3] ^ ks[j + 10]; + + if (++i == 7) break; + + temp = ks[j + 11]; + + ks[j + 12] = ks[j + 4] + ^ (te2[(temp >> 24) & 0xff] & 0xff000000) + ^ (te3[(temp >> 16) & 0xff] & 0x00ff0000) + ^ (te0[(temp >> 8) & 0xff] & 0x0000ff00) + ^ (te1[(temp >> 0) & 0xff] & 0x000000ff); + + ks[j + 13] = ks[j + 5] ^ ks[j + 12]; + ks[j + 14] = ks[j + 6] ^ ks[j + 13]; + ks[j + 15] = ks[j + 7] ^ ks[j + 14]; + + j += 8; + } +} + +void aes256_InvertKey (u32 *ks) +{ + for (u32 i = 0, j = 56; i < j; i += 4, j -= 4) + { + u32 temp; + + temp = ks[i + 0]; ks[i + 0] = ks[j + 0]; ks[j + 0] = temp; + temp = ks[i + 1]; ks[i + 1] = ks[j + 1]; ks[j + 1] = temp; + temp = ks[i + 2]; ks[i + 2] = ks[j + 2]; ks[j + 2] = temp; + temp = ks[i + 3]; ks[i + 3] = ks[j + 3]; ks[j + 3] = temp; + } + + for (u32 i = 1, j = 4; i < 14; i += 1, j += 4) + { + ks[j + 0] = + td0[te1[(ks[j + 0] >> 24) & 0xff] & 0xff] ^ + td1[te1[(ks[j + 0] >> 16) & 0xff] & 0xff] ^ + td2[te1[(ks[j + 0] >> 8) & 0xff] & 0xff] ^ + td3[te1[(ks[j + 0] >> 0) & 0xff] & 0xff]; + + ks[j + 1] = + td0[te1[(ks[j + 1] >> 24) & 0xff] & 0xff] ^ + td1[te1[(ks[j + 1] >> 16) & 0xff] & 0xff] ^ + td2[te1[(ks[j + 1] >> 8) & 0xff] & 0xff] ^ + td3[te1[(ks[j + 1] >> 0) & 0xff] & 0xff]; + + ks[j + 2] = + td0[te1[(ks[j + 2] >> 24) & 0xff] & 0xff] ^ + td1[te1[(ks[j + 2] >> 16) & 0xff] & 0xff] ^ + td2[te1[(ks[j + 2] >> 8) & 0xff] & 0xff] ^ + td3[te1[(ks[j + 2] >> 0) & 0xff] & 0xff]; + + ks[j + 3] = + td0[te1[(ks[j + 3] >> 24) & 0xff] & 0xff] ^ + td1[te1[(ks[j + 3] >> 16) & 0xff] & 0xff] ^ + td2[te1[(ks[j + 3] >> 8) & 0xff] & 0xff] ^ + td3[te1[(ks[j + 3] >> 0) & 0xff] & 0xff]; + } +} + +void aes256_set_encrypt_key (u32 *ks, const u32 *ukey) +{ + u32 ukey_s[8]; + + ukey_s[0] = swap32 (ukey[0]); + ukey_s[1] = swap32 (ukey[1]); + ukey_s[2] = swap32 (ukey[2]); + ukey_s[3] = swap32 (ukey[3]); + ukey_s[4] = swap32 (ukey[4]); + ukey_s[5] = swap32 (ukey[5]); + ukey_s[6] = swap32 (ukey[6]); + ukey_s[7] = swap32 (ukey[7]); + + aes256_ExpandKey (ks, ukey_s); +} + +void aes256_set_decrypt_key (u32 *ks, const u32 *ukey) +{ + u32 ukey_s[8]; + + ukey_s[0] = swap32 (ukey[0]); + ukey_s[1] = swap32 (ukey[1]); + ukey_s[2] = swap32 (ukey[2]); + ukey_s[3] = swap32 (ukey[3]); + ukey_s[4] = swap32 (ukey[4]); + ukey_s[5] = swap32 (ukey[5]); + ukey_s[6] = swap32 (ukey[6]); + ukey_s[7] = swap32 (ukey[7]); + + aes256_ExpandKey (ks, ukey_s); + + aes256_InvertKey (ks); +} + +void aes256_decrypt (const u32 *ks, const u32 *in, u32 *out) +{ + u32 in_s[4]; + + in_s[0] = swap32 (in[0]); + in_s[1] = swap32 (in[1]); + in_s[2] = swap32 (in[2]); + in_s[3] = swap32 (in[3]); + + u32 s0 = in_s[0] ^ ks[0]; + u32 s1 = in_s[1] ^ ks[1]; + u32 s2 = in_s[2] ^ ks[2]; + u32 s3 = in_s[3] ^ ks[3]; + + u32 t0; + u32 t1; + u32 t2; + u32 t3; + + t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[ 4]; + t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[ 5]; + t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[ 6]; + t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[ 7]; + s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[ 8]; + s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[ 9]; + s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[10]; + s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[11]; + t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[12]; + t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[13]; + t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[14]; + t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[15]; + s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[16]; + s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[17]; + s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[18]; + s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[19]; + t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[20]; + t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[21]; + t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[22]; + t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[23]; + s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[24]; + s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[25]; + s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[26]; + s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[27]; + t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[28]; + t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[29]; + t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[30]; + t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[31]; + s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[32]; + s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[33]; + s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[34]; + s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[35]; + t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[36]; + t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[37]; + t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[38]; + t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[39]; + s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[40]; + s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[41]; + s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[42]; + s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[43]; + t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[44]; + t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[45]; + t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[46]; + t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[47]; + s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[48]; + s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[49]; + s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[50]; + s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[51]; + t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[52]; + t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[53]; + t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[54]; + t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[55]; + + out[0] = (td4[(t0 >> 24) & 0xff] & 0xff000000) + ^ (td4[(t3 >> 16) & 0xff] & 0x00ff0000) + ^ (td4[(t2 >> 8) & 0xff] & 0x0000ff00) + ^ (td4[(t1 >> 0) & 0xff] & 0x000000ff) + ^ ks[56]; + + out[1] = (td4[(t1 >> 24) & 0xff] & 0xff000000) + ^ (td4[(t0 >> 16) & 0xff] & 0x00ff0000) + ^ (td4[(t3 >> 8) & 0xff] & 0x0000ff00) + ^ (td4[(t2 >> 0) & 0xff] & 0x000000ff) + ^ ks[57]; + + out[2] = (td4[(t2 >> 24) & 0xff] & 0xff000000) + ^ (td4[(t1 >> 16) & 0xff] & 0x00ff0000) + ^ (td4[(t0 >> 8) & 0xff] & 0x0000ff00) + ^ (td4[(t3 >> 0) & 0xff] & 0x000000ff) + ^ ks[58]; + + out[3] = (td4[(t3 >> 24) & 0xff] & 0xff000000) + ^ (td4[(t2 >> 16) & 0xff] & 0x00ff0000) + ^ (td4[(t1 >> 8) & 0xff] & 0x0000ff00) + ^ (td4[(t0 >> 0) & 0xff] & 0x000000ff) + ^ ks[59]; + + out[0] = swap32 (out[0]); + out[1] = swap32 (out[1]); + out[2] = swap32 (out[2]); + out[3] = swap32 (out[3]); +} + +void aes256_encrypt (const u32 *ks, const u32 *in, u32 *out) +{ + u32 in_s[4]; + + in_s[0] = swap32 (in[0]); + in_s[1] = swap32 (in[1]); + in_s[2] = swap32 (in[2]); + in_s[3] = swap32 (in[3]); + + u32 s0 = in_s[0] ^ ks[0]; + u32 s1 = in_s[1] ^ ks[1]; + u32 s2 = in_s[2] ^ ks[2]; + u32 s3 = in_s[3] ^ ks[3]; + + u32 t0; + u32 t1; + u32 t2; + u32 t3; + + t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[ 4]; + t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[ 5]; + t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[ 6]; + t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[ 7]; + s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[ 8]; + s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[ 9]; + s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[10]; + s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[11]; + t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[12]; + t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[13]; + t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[14]; + t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[15]; + s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[16]; + s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[17]; + s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[18]; + s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[19]; + t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[20]; + t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[21]; + t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[22]; + t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[23]; + s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[24]; + s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[25]; + s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[26]; + s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[27]; + t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[28]; + t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[29]; + t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[30]; + t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[31]; + s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[32]; + s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[33]; + s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[34]; + s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[35]; + t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[36]; + t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[37]; + t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[38]; + t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[39]; + s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[40]; + s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[41]; + s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[42]; + s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[43]; + t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[44]; + t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[45]; + t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[46]; + t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[47]; + s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[48]; + s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[49]; + s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[50]; + s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[51]; + t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[52]; + t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[53]; + t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[54]; + t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[55]; + + out[0] = (te4[(t0 >> 24) & 0xff] & 0xff000000) + ^ (te4[(t1 >> 16) & 0xff] & 0x00ff0000) + ^ (te4[(t2 >> 8) & 0xff] & 0x0000ff00) + ^ (te4[(t3 >> 0) & 0xff] & 0x000000ff) + ^ ks[56]; + + out[1] = (te4[(t1 >> 24) & 0xff] & 0xff000000) + ^ (te4[(t2 >> 16) & 0xff] & 0x00ff0000) + ^ (te4[(t3 >> 8) & 0xff] & 0x0000ff00) + ^ (te4[(t0 >> 0) & 0xff] & 0x000000ff) + ^ ks[57]; + + out[2] = (te4[(t2 >> 24) & 0xff] & 0xff000000) + ^ (te4[(t3 >> 16) & 0xff] & 0x00ff0000) + ^ (te4[(t0 >> 8) & 0xff] & 0x0000ff00) + ^ (te4[(t1 >> 0) & 0xff] & 0x000000ff) + ^ ks[58]; + + out[3] = (te4[(t3 >> 24) & 0xff] & 0xff000000) + ^ (te4[(t0 >> 16) & 0xff] & 0x00ff0000) + ^ (te4[(t1 >> 8) & 0xff] & 0x0000ff00) + ^ (te4[(t2 >> 0) & 0xff] & 0x000000ff) + ^ ks[59]; + + out[0] = swap32 (out[0]); + out[1] = swap32 (out[1]); + out[2] = swap32 (out[2]); + out[3] = swap32 (out[3]); +} + +void aes256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out) +{ + u32 T[4] = { 0 }; + u32 Z[4] = { 0 }; + + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; + + u32 ks[60]; + + aes256_set_encrypt_key (ks, ukey2); + aes256_encrypt (ks, Z, T); + + out[0] ^= T[0]; + out[1] ^= T[1]; + out[2] ^= T[2]; + out[3] ^= T[3]; + + aes256_set_decrypt_key (ks, ukey1); + aes256_decrypt (ks, out, out); + + out[0] ^= T[0]; + out[1] ^= T[1]; + out[2] ^= T[2]; + out[3] ^= T[3]; +} diff --git a/OpenCL/inc_cipher_serpent256.cl b/OpenCL/inc_cipher_serpent256.cl new file mode 100644 index 0000000..462c0b9 --- /dev/null +++ b/OpenCL/inc_cipher_serpent256.cl @@ -0,0 +1,587 @@ +/* This is an independent implementation of the encryption algorithm: */ +/* */ +/* Serpent by Ross Anderson, Eli Biham and Lars Knudsen */ +/* */ +/* which is a candidate algorithm in the Advanced Encryption Standard */ +/* programme of the US National Institute of Standards and Technology. */ +/* */ +/* Copyright in this implementation is held by Dr B R Gladman but I */ +/* hereby give permission for its free direct or derivative use subject */ +/* to acknowledgment of its origin and compliance with any conditions */ +/* that the originators of the algorithm place on its exploitation. */ +/* */ +/* Dr Brian Gladman (gladman@seven77.demon.co.uk) 14th January 1999 */ +/* */ +/* -------------------------------------------------------------------- */ +/* */ +/* Cleaned and optimized for GPU use with hashcat by Jens Steube */ + +/* 15 terms */ + +#define sb0(a,b,c,d,e,f,g,h) \ + t1 = a ^ d; \ + t2 = a & d; \ + t3 = c ^ t1; \ + t6 = b & t1; \ + t4 = b ^ t3; \ + t10 = ~t3; \ + h = t2 ^ t4; \ + t7 = a ^ t6; \ + t14 = ~t7; \ + t8 = c | t7; \ + t11 = t3 ^ t7; \ + g = t4 ^ t8; \ + t12 = h & t11; \ + f = t10 ^ t12; \ + e = t12 ^ t14 + +/* 15 terms */ + +#define ib0(a,b,c,d,e,f,g,h) \ + t1 = ~a; \ + t2 = a ^ b; \ + t3 = t1 | t2; \ + t4 = d ^ t3; \ + t7 = d & t2; \ + t5 = c ^ t4; \ + t8 = t1 ^ t7; \ + g = t2 ^ t5; \ + t11 = a & t4; \ + t9 = g & t8; \ + t14 = t5 ^ t8; \ + f = t4 ^ t9; \ + t12 = t5 | f; \ + h = t11 ^ t12; \ + e = h ^ t14 + +/* 14 terms! */ + +#define sb1(a,b,c,d,e,f,g,h) \ + t1 = ~a; \ + t2 = b ^ t1; \ + t3 = a | t2; \ + t4 = d | t2; \ + t5 = c ^ t3; \ + g = d ^ t5; \ + t7 = b ^ t4; \ + t8 = t2 ^ g; \ + t9 = t5 & t7; \ + h = t8 ^ t9; \ + t11 = t5 ^ t7; \ + f = h ^ t11; \ + t13 = t8 & t11; \ + e = t5 ^ t13 + +/* 17 terms */ + +#define ib1(a,b,c,d,e,f,g,h) \ + t1 = a ^ d; \ + t2 = a & b; \ + t3 = b ^ c; \ + t4 = a ^ t3; \ + t5 = b | d; \ + t7 = c | t1; \ + h = t4 ^ t5; \ + t8 = b ^ t7; \ + t11 = ~t2; \ + t9 = t4 & t8; \ + f = t1 ^ t9; \ + t13 = t9 ^ t11; \ + t12 = h & f; \ + g = t12 ^ t13; \ + t15 = a & d; \ + t16 = c ^ t13; \ + e = t15 ^ t16 + +/* 16 terms */ + +#define sb2(a,b,c,d,e,f,g,h) \ + t1 = ~a; \ + t2 = b ^ d; \ + t3 = c & t1; \ + t13 = d | t1; \ + e = t2 ^ t3; \ + t5 = c ^ t1; \ + t6 = c ^ e; \ + t7 = b & t6; \ + t10 = e | t5; \ + h = t5 ^ t7; \ + t9 = d | t7; \ + t11 = t9 & t10; \ + t14 = t2 ^ h; \ + g = a ^ t11; \ + t15 = g ^ t13; \ + f = t14 ^ t15 + +/* 16 terms */ + +#define ib2(a,b,c,d,e,f,g,h) \ + t1 = b ^ d; \ + t2 = ~t1; \ + t3 = a ^ c; \ + t4 = c ^ t1; \ + t7 = a | t2; \ + t5 = b & t4; \ + t8 = d ^ t7; \ + t11 = ~t4; \ + e = t3 ^ t5; \ + t9 = t3 | t8; \ + t14 = d & t11; \ + h = t1 ^ t9; \ + t12 = e | h; \ + f = t11 ^ t12; \ + t15 = t3 ^ t12; \ + g = t14 ^ t15 + +/* 17 terms */ + +#define sb3(a,b,c,d,e,f,g,h) \ + t1 = a ^ c; \ + t2 = d ^ t1; \ + t3 = a & t2; \ + t4 = d ^ t3; \ + t5 = b & t4; \ + g = t2 ^ t5; \ + t7 = a | g; \ + t8 = b | d; \ + t11 = a | d; \ + t9 = t4 & t7; \ + f = t8 ^ t9; \ + t12 = b ^ t11; \ + t13 = g ^ t9; \ + t15 = t3 ^ t8; \ + h = t12 ^ t13; \ + t16 = c & t15; \ + e = t12 ^ t16 + +/* 16 term solution that performs less well than 17 term one + in my environment (PPro/PII) + +#define sb3(a,b,c,d,e,f,g,h) \ + t1 = a ^ b; \ + t2 = a & c; \ + t3 = a | d; \ + t4 = c ^ d; \ + t5 = t1 & t3; \ + t6 = t2 | t5; \ + g = t4 ^ t6; \ + t8 = b ^ t3; \ + t9 = t6 ^ t8; \ + t10 = t4 & t9; \ + e = t1 ^ t10; \ + t12 = g & e; \ + f = t9 ^ t12; \ + t14 = b | d; \ + t15 = t4 ^ t12; \ + h = t14 ^ t15 +*/ + +/* 17 terms */ + +#define ib3(a,b,c,d,e,f,g,h) \ + t1 = b ^ c; \ + t2 = b | c; \ + t3 = a ^ c; \ + t7 = a ^ d; \ + t4 = t2 ^ t3; \ + t5 = d | t4; \ + t9 = t2 ^ t7; \ + e = t1 ^ t5; \ + t8 = t1 | t5; \ + t11 = a & t4; \ + g = t8 ^ t9; \ + t12 = e | t9; \ + f = t11 ^ t12; \ + t14 = a & g; \ + t15 = t2 ^ t14; \ + t16 = e & t15; \ + h = t4 ^ t16 + +/* 15 terms */ + +#define sb4(a,b,c,d,e,f,g,h) \ + t1 = a ^ d; \ + t2 = d & t1; \ + t3 = c ^ t2; \ + t4 = b | t3; \ + h = t1 ^ t4; \ + t6 = ~b; \ + t7 = t1 | t6; \ + e = t3 ^ t7; \ + t9 = a & e; \ + t10 = t1 ^ t6; \ + t11 = t4 & t10; \ + g = t9 ^ t11; \ + t13 = a ^ t3; \ + t14 = t10 & g; \ + f = t13 ^ t14 + +/* 17 terms */ + +#define ib4(a,b,c,d,e,f,g,h) \ + t1 = c ^ d; \ + t2 = c | d; \ + t3 = b ^ t2; \ + t4 = a & t3; \ + f = t1 ^ t4; \ + t6 = a ^ d; \ + t7 = b | d; \ + t8 = t6 & t7; \ + h = t3 ^ t8; \ + t10 = ~a; \ + t11 = c ^ h; \ + t12 = t10 | t11;\ + e = t3 ^ t12; \ + t14 = c | t4; \ + t15 = t7 ^ t14; \ + t16 = h | t10; \ + g = t15 ^ t16 + +/* 16 terms */ + +#define sb5(a,b,c,d,e,f,g,h) \ + t1 = ~a; \ + t2 = a ^ b; \ + t3 = a ^ d; \ + t4 = c ^ t1; \ + t5 = t2 | t3; \ + e = t4 ^ t5; \ + t7 = d & e; \ + t8 = t2 ^ e; \ + t10 = t1 | e; \ + f = t7 ^ t8; \ + t11 = t2 | t7; \ + t12 = t3 ^ t10; \ + t14 = b ^ t7; \ + g = t11 ^ t12; \ + t15 = f & t12; \ + h = t14 ^ t15 + +/* 16 terms */ + +#define ib5(a,b,c,d,e,f,g,h) \ + t1 = ~c; \ + t2 = b & t1; \ + t3 = d ^ t2; \ + t4 = a & t3; \ + t5 = b ^ t1; \ + h = t4 ^ t5; \ + t7 = b | h; \ + t8 = a & t7; \ + f = t3 ^ t8; \ + t10 = a | d; \ + t11 = t1 ^ t7; \ + e = t10 ^ t11; \ + t13 = a ^ c; \ + t14 = b & t10; \ + t15 = t4 | t13; \ + g = t14 ^ t15 + +/* 15 terms */ + +#define sb6(a,b,c,d,e,f,g,h) \ + t1 = ~a; \ + t2 = a ^ d; \ + t3 = b ^ t2; \ + t4 = t1 | t2; \ + t5 = c ^ t4; \ + f = b ^ t5; \ + t13 = ~t5; \ + t7 = t2 | f; \ + t8 = d ^ t7; \ + t9 = t5 & t8; \ + g = t3 ^ t9; \ + t11 = t5 ^ t8; \ + e = g ^ t11; \ + t14 = t3 & t11; \ + h = t13 ^ t14 + +/* 15 terms */ + +#define ib6(a,b,c,d,e,f,g,h) \ + t1 = ~a; \ + t2 = a ^ b; \ + t3 = c ^ t2; \ + t4 = c | t1; \ + t5 = d ^ t4; \ + t13 = d & t1; \ + f = t3 ^ t5; \ + t7 = t3 & t5; \ + t8 = t2 ^ t7; \ + t9 = b | t8; \ + h = t5 ^ t9; \ + t11 = b | h; \ + e = t8 ^ t11; \ + t14 = t3 ^ t11; \ + g = t13 ^ t14 + +/* 17 terms */ + +#define sb7(a,b,c,d,e,f,g,h) \ + t1 = ~c; \ + t2 = b ^ c; \ + t3 = b | t1; \ + t4 = d ^ t3; \ + t5 = a & t4; \ + t7 = a ^ d; \ + h = t2 ^ t5; \ + t8 = b ^ t5; \ + t9 = t2 | t8; \ + t11 = d & t3; \ + f = t7 ^ t9; \ + t12 = t5 ^ f; \ + t15 = t1 | t4; \ + t13 = h & t12; \ + g = t11 ^ t13; \ + t16 = t12 ^ g; \ + e = t15 ^ t16 + +/* 17 terms */ + +#define ib7(a,b,c,d,e,f,g,h) \ + t1 = a & b; \ + t2 = a | b; \ + t3 = c | t1; \ + t4 = d & t2; \ + h = t3 ^ t4; \ + t6 = ~d; \ + t7 = b ^ t4; \ + t8 = h ^ t6; \ + t11 = c ^ t7; \ + t9 = t7 | t8; \ + f = a ^ t9; \ + t12 = d | f; \ + e = t11 ^ t12; \ + t14 = a & h; \ + t15 = t3 ^ f; \ + t16 = e ^ t14; \ + g = t15 ^ t16 + +#define k_xor(r,a,b,c,d) \ + a ^= ks[4 * r + 8]; \ + b ^= ks[4 * r + 9]; \ + c ^= ks[4 * r + 10]; \ + d ^= ks[4 * r + 11] + +#define k_set(r,a,b,c,d) \ + a = ks[4 * r + 8]; \ + b = ks[4 * r + 9]; \ + c = ks[4 * r + 10]; \ + d = ks[4 * r + 11] + +#define k_get(r,a,b,c,d) \ + ks[4 * r + 8] = a; \ + ks[4 * r + 9] = b; \ + ks[4 * r + 10] = c; \ + ks[4 * r + 11] = d + +/* the linear transformation and its inverse */ + +#define rot(a,b,c,d) \ + a = rotl32(a, 13); \ + c = rotl32(c, 3); \ + d ^= c ^ (a << 3); \ + b ^= a ^ c; \ + d = rotl32(d, 7); \ + b = rotl32(b, 1); \ + a ^= b ^ d; \ + c ^= d ^ (b << 7); \ + a = rotl32(a, 5); \ + c = rotl32(c, 22) + +#define irot(a,b,c,d) \ + c = rotr32(c, 22); \ + a = rotr32(a, 5); \ + c ^= d ^ (b << 7); \ + a ^= b ^ d; \ + d = rotr32(d, 7); \ + b = rotr32(b, 1); \ + d ^= c ^ (a << 3); \ + b ^= a ^ c; \ + c = rotr32(c, 3); \ + a = rotr32(a, 13) + +void serpent256_set_key (u32 *ks, const u32 *ukey) +{ + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + ks[i] = ukey[i]; + } + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 132; i++) + { + ks[i + 8] = rotl32 (ks[i + 7] ^ ks[i + 5] ^ ks[i + 3] ^ ks[i + 0] ^ 0x9e3779b9 ^ i, 11); + } + + u32 a,b,c,d,e,f,g,h; + u32 t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16; + + k_set( 0,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get( 0,e,f,g,h); + k_set( 1,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get( 1,e,f,g,h); + k_set( 2,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get( 2,e,f,g,h); + k_set( 3,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get( 3,e,f,g,h); + k_set( 4,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get( 4,e,f,g,h); + k_set( 5,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get( 5,e,f,g,h); + k_set( 6,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get( 6,e,f,g,h); + k_set( 7,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get( 7,e,f,g,h); + k_set( 8,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get( 8,e,f,g,h); + k_set( 9,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get( 9,e,f,g,h); + k_set(10,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(10,e,f,g,h); + k_set(11,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(11,e,f,g,h); + k_set(12,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(12,e,f,g,h); + k_set(13,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(13,e,f,g,h); + k_set(14,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(14,e,f,g,h); + k_set(15,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(15,e,f,g,h); + k_set(16,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(16,e,f,g,h); + k_set(17,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get(17,e,f,g,h); + k_set(18,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(18,e,f,g,h); + k_set(19,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(19,e,f,g,h); + k_set(20,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(20,e,f,g,h); + k_set(21,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(21,e,f,g,h); + k_set(22,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(22,e,f,g,h); + k_set(23,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(23,e,f,g,h); + k_set(24,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(24,e,f,g,h); + k_set(25,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get(25,e,f,g,h); + k_set(26,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(26,e,f,g,h); + k_set(27,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(27,e,f,g,h); + k_set(28,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(28,e,f,g,h); + k_set(29,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(29,e,f,g,h); + k_set(30,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(30,e,f,g,h); + k_set(31,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(31,e,f,g,h); + k_set(32,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(32,e,f,g,h); +} + +void serpent256_encrypt (const u32 *ks, const u32 *in, u32 *out) +{ + u32 a,b,c,d,e,f,g,h; + u32 t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16; + + a = in[0]; + b = in[1]; + c = in[2]; + d = in[3]; + + k_xor( 0,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor( 1,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor( 2,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor( 3,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor( 4,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor( 5,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor( 6,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor( 7,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor( 8,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor( 9,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(10,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(11,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(12,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(13,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(14,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(15,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(16,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(17,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(18,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(19,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(20,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(21,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(22,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(23,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(24,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(25,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(26,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(27,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(28,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(29,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d); + k_xor(30,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h); + k_xor(31,e,f,g,h); sb7(e,f,g,h,a,b,c,d); + k_xor(32,a,b,c,d); + + out[0] = a; + out[1] = b; + out[2] = c; + out[3] = d; +} + +void serpent256_decrypt (const u32 *ks, const u32 *in, u32 *out) +{ + u32 a,b,c,d,e,f,g,h; + u32 t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16; + + a = in[0]; + b = in[1]; + c = in[2]; + d = in[3]; + + k_xor(32,a,b,c,d); + ib7(a,b,c,d,e,f,g,h); k_xor(31,e,f,g,h); + irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(30,a,b,c,d); + irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(29,e,f,g,h); + irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(28,a,b,c,d); + irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(27,e,f,g,h); + irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(26,a,b,c,d); + irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor(25,e,f,g,h); + irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor(24,a,b,c,d); + irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor(23,e,f,g,h); + irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(22,a,b,c,d); + irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(21,e,f,g,h); + irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(20,a,b,c,d); + irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(19,e,f,g,h); + irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(18,a,b,c,d); + irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor(17,e,f,g,h); + irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor(16,a,b,c,d); + irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor(15,e,f,g,h); + irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(14,a,b,c,d); + irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(13,e,f,g,h); + irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(12,a,b,c,d); + irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(11,e,f,g,h); + irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(10,a,b,c,d); + irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor( 9,e,f,g,h); + irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor( 8,a,b,c,d); + irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor( 7,e,f,g,h); + irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor( 6,a,b,c,d); + irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor( 5,e,f,g,h); + irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor( 4,a,b,c,d); + irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor( 3,e,f,g,h); + irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor( 2,a,b,c,d); + irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor( 1,e,f,g,h); + irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor( 0,a,b,c,d); + + out[0] = a; + out[1] = b; + out[2] = c; + out[3] = d; +} + +void serpent256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out) +{ + u32 T[4] = { 0 }; + u32 Z[4] = { 0 }; + + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; + + u32 ks[140]; + + serpent256_set_key (ks, ukey2); + serpent256_encrypt (ks, Z, T); + + out[0] ^= T[0]; + out[1] ^= T[1]; + out[2] ^= T[2]; + out[3] ^= T[3]; + + serpent256_set_key (ks, ukey1); + serpent256_decrypt (ks, out, out); + + out[0] ^= T[0]; + out[1] ^= T[1]; + out[2] ^= T[2]; + out[3] ^= T[3]; +} diff --git a/OpenCL/inc_cipher_twofish256.cl b/OpenCL/inc_cipher_twofish256.cl new file mode 100644 index 0000000..6482a63 --- /dev/null +++ b/OpenCL/inc_cipher_twofish256.cl @@ -0,0 +1,466 @@ +/* This is an independent implementation of the encryption algorithm: */ +/* */ +/* Twofish by Bruce Schneier and colleagues */ +/* */ +/* which is a candidate algorithm in the Advanced Encryption Standard */ +/* programme of the US National Institute of Standards and Technology. */ +/* */ +/* Copyright in this implementation is held by Dr B R Gladman but I */ +/* hereby give permission for its free direct or derivative use subject */ +/* to acknowledgment of its origin and compliance with any conditions */ +/* that the originators of t he algorithm place on its exploitation. */ +/* */ +/* My thanks to Doug Whiting and Niels Ferguson for comments that led */ +/* to improvements in this implementation. */ +/* */ +/* Dr Brian Gladman (gladman@seven77.demon.co.uk) 14th January 1999 */ +/* */ +/* -------------------------------------------------------------------- */ +/* */ +/* Cleaned and optimized for GPU use with hashcat by Jens Steube */ + +#define extract_byte(x,n) (((x) >> (8 * (n))) & 0xff) + +__constant u32 q_tab[2][256] = +{ + { + 0xA9, 0x67, 0xB3, 0xE8, 0x04, 0xFD, 0xA3, 0x76, 0x9A, 0x92, 0x80, 0x78, + 0xE4, 0xDD, 0xD1, 0x38, 0x0D, 0xC6, 0x35, 0x98, 0x18, 0xF7, 0xEC, 0x6C, + 0x43, 0x75, 0x37, 0x26, 0xFA, 0x13, 0x94, 0x48, 0xF2, 0xD0, 0x8B, 0x30, + 0x84, 0x54, 0xDF, 0x23, 0x19, 0x5B, 0x3D, 0x59, 0xF3, 0xAE, 0xA2, 0x82, + 0x63, 0x01, 0x83, 0x2E, 0xD9, 0x51, 0x9B, 0x7C, 0xA6, 0xEB, 0xA5, 0xBE, + 0x16, 0x0C, 0xE3, 0x61, 0xC0, 0x8C, 0x3A, 0xF5, 0x73, 0x2C, 0x25, 0x0B, + 0xBB, 0x4E, 0x89, 0x6B, 0x53, 0x6A, 0xB4, 0xF1, 0xE1, 0xE6, 0xBD, 0x45, + 0xE2, 0xF4, 0xB6, 0x66, 0xCC, 0x95, 0x03, 0x56, 0xD4, 0x1C, 0x1E, 0xD7, + 0xFB, 0xC3, 0x8E, 0xB5, 0xE9, 0xCF, 0xBF, 0xBA, 0xEA, 0x77, 0x39, 0xAF, + 0x33, 0xC9, 0x62, 0x71, 0x81, 0x79, 0x09, 0xAD, 0x24, 0xCD, 0xF9, 0xD8, + 0xE5, 0xC5, 0xB9, 0x4D, 0x44, 0x08, 0x86, 0xE7, 0xA1, 0x1D, 0xAA, 0xED, + 0x06, 0x70, 0xB2, 0xD2, 0x41, 0x7B, 0xA0, 0x11, 0x31, 0xC2, 0x27, 0x90, + 0x20, 0xF6, 0x60, 0xFF, 0x96, 0x5C, 0xB1, 0xAB, 0x9E, 0x9C, 0x52, 0x1B, + 0x5F, 0x93, 0x0A, 0xEF, 0x91, 0x85, 0x49, 0xEE, 0x2D, 0x4F, 0x8F, 0x3B, + 0x47, 0x87, 0x6D, 0x46, 0xD6, 0x3E, 0x69, 0x64, 0x2A, 0xCE, 0xCB, 0x2F, + 0xFC, 0x97, 0x05, 0x7A, 0xAC, 0x7F, 0xD5, 0x1A, 0x4B, 0x0E, 0xA7, 0x5A, + 0x28, 0x14, 0x3F, 0x29, 0x88, 0x3C, 0x4C, 0x02, 0xB8, 0xDA, 0xB0, 0x17, + 0x55, 0x1F, 0x8A, 0x7D, 0x57, 0xC7, 0x8D, 0x74, 0xB7, 0xC4, 0x9F, 0x72, + 0x7E, 0x15, 0x22, 0x12, 0x58, 0x07, 0x99, 0x34, 0x6E, 0x50, 0xDE, 0x68, + 0x65, 0xBC, 0xDB, 0xF8, 0xC8, 0xA8, 0x2B, 0x40, 0xDC, 0xFE, 0x32, 0xA4, + 0xCA, 0x10, 0x21, 0xF0, 0xD3, 0x5D, 0x0F, 0x00, 0x6F, 0x9D, 0x36, 0x42, + 0x4A, 0x5E, 0xC1, 0xE0 + }, + { + 0x75, 0xF3, 0xC6, 0xF4, 0xDB, 0x7B, 0xFB, 0xC8, 0x4A, 0xD3, 0xE6, 0x6B, + 0x45, 0x7D, 0xE8, 0x4B, 0xD6, 0x32, 0xD8, 0xFD, 0x37, 0x71, 0xF1, 0xE1, + 0x30, 0x0F, 0xF8, 0x1B, 0x87, 0xFA, 0x06, 0x3F, 0x5E, 0xBA, 0xAE, 0x5B, + 0x8A, 0x00, 0xBC, 0x9D, 0x6D, 0xC1, 0xB1, 0x0E, 0x80, 0x5D, 0xD2, 0xD5, + 0xA0, 0x84, 0x07, 0x14, 0xB5, 0x90, 0x2C, 0xA3, 0xB2, 0x73, 0x4C, 0x54, + 0x92, 0x74, 0x36, 0x51, 0x38, 0xB0, 0xBD, 0x5A, 0xFC, 0x60, 0x62, 0x96, + 0x6C, 0x42, 0xF7, 0x10, 0x7C, 0x28, 0x27, 0x8C, 0x13, 0x95, 0x9C, 0xC7, + 0x24, 0x46, 0x3B, 0x70, 0xCA, 0xE3, 0x85, 0xCB, 0x11, 0xD0, 0x93, 0xB8, + 0xA6, 0x83, 0x20, 0xFF, 0x9F, 0x77, 0xC3, 0xCC, 0x03, 0x6F, 0x08, 0xBF, + 0x40, 0xE7, 0x2B, 0xE2, 0x79, 0x0C, 0xAA, 0x82, 0x41, 0x3A, 0xEA, 0xB9, + 0xE4, 0x9A, 0xA4, 0x97, 0x7E, 0xDA, 0x7A, 0x17, 0x66, 0x94, 0xA1, 0x1D, + 0x3D, 0xF0, 0xDE, 0xB3, 0x0B, 0x72, 0xA7, 0x1C, 0xEF, 0xD1, 0x53, 0x3E, + 0x8F, 0x33, 0x26, 0x5F, 0xEC, 0x76, 0x2A, 0x49, 0x81, 0x88, 0xEE, 0x21, + 0xC4, 0x1A, 0xEB, 0xD9, 0xC5, 0x39, 0x99, 0xCD, 0xAD, 0x31, 0x8B, 0x01, + 0x18, 0x23, 0xDD, 0x1F, 0x4E, 0x2D, 0xF9, 0x48, 0x4F, 0xF2, 0x65, 0x8E, + 0x78, 0x5C, 0x58, 0x19, 0x8D, 0xE5, 0x98, 0x57, 0x67, 0x7F, 0x05, 0x64, + 0xAF, 0x63, 0xB6, 0xFE, 0xF5, 0xB7, 0x3C, 0xA5, 0xCE, 0xE9, 0x68, 0x44, + 0xE0, 0x4D, 0x43, 0x69, 0x29, 0x2E, 0xAC, 0x15, 0x59, 0xA8, 0x0A, 0x9E, + 0x6E, 0x47, 0xDF, 0x34, 0x35, 0x6A, 0xCF, 0xDC, 0x22, 0xC9, 0xC0, 0x9B, + 0x89, 0xD4, 0xED, 0xAB, 0x12, 0xA2, 0x0D, 0x52, 0xBB, 0x02, 0x2F, 0xA9, + 0xD7, 0x61, 0x1E, 0xB4, 0x50, 0x04, 0xF6, 0xC2, 0x16, 0x25, 0x86, 0x56, + 0x55, 0x09, 0xBE, 0x91 + } +}; + +#define q(n,x) q_tab[n][x] + +__constant u32 m_tab[4][256] = +{ + { 0xBCBC3275, 0xECEC21F3, 0x202043C6, 0xB3B3C9F4, 0xDADA03DB, 0x02028B7B, + 0xE2E22BFB, 0x9E9EFAC8, 0xC9C9EC4A, 0xD4D409D3, 0x18186BE6, 0x1E1E9F6B, + 0x98980E45, 0xB2B2387D, 0xA6A6D2E8, 0x2626B74B, 0x3C3C57D6, 0x93938A32, + 0x8282EED8, 0x525298FD, 0x7B7BD437, 0xBBBB3771, 0x5B5B97F1, 0x474783E1, + 0x24243C30, 0x5151E20F, 0xBABAC6F8, 0x4A4AF31B, 0xBFBF4887, 0x0D0D70FA, + 0xB0B0B306, 0x7575DE3F, 0xD2D2FD5E, 0x7D7D20BA, 0x666631AE, 0x3A3AA35B, + 0x59591C8A, 0x00000000, 0xCDCD93BC, 0x1A1AE09D, 0xAEAE2C6D, 0x7F7FABC1, + 0x2B2BC7B1, 0xBEBEB90E, 0xE0E0A080, 0x8A8A105D, 0x3B3B52D2, 0x6464BAD5, + 0xD8D888A0, 0xE7E7A584, 0x5F5FE807, 0x1B1B1114, 0x2C2CC2B5, 0xFCFCB490, + 0x3131272C, 0x808065A3, 0x73732AB2, 0x0C0C8173, 0x79795F4C, 0x6B6B4154, + 0x4B4B0292, 0x53536974, 0x94948F36, 0x83831F51, 0x2A2A3638, 0xC4C49CB0, + 0x2222C8BD, 0xD5D5F85A, 0xBDBDC3FC, 0x48487860, 0xFFFFCE62, 0x4C4C0796, + 0x4141776C, 0xC7C7E642, 0xEBEB24F7, 0x1C1C1410, 0x5D5D637C, 0x36362228, + 0x6767C027, 0xE9E9AF8C, 0x4444F913, 0x1414EA95, 0xF5F5BB9C, 0xCFCF18C7, + 0x3F3F2D24, 0xC0C0E346, 0x7272DB3B, 0x54546C70, 0x29294CCA, 0xF0F035E3, + 0x0808FE85, 0xC6C617CB, 0xF3F34F11, 0x8C8CE4D0, 0xA4A45993, 0xCACA96B8, + 0x68683BA6, 0xB8B84D83, 0x38382820, 0xE5E52EFF, 0xADAD569F, 0x0B0B8477, + 0xC8C81DC3, 0x9999FFCC, 0x5858ED03, 0x19199A6F, 0x0E0E0A08, 0x95957EBF, + 0x70705040, 0xF7F730E7, 0x6E6ECF2B, 0x1F1F6EE2, 0xB5B53D79, 0x09090F0C, + 0x616134AA, 0x57571682, 0x9F9F0B41, 0x9D9D803A, 0x111164EA, 0x2525CDB9, + 0xAFAFDDE4, 0x4545089A, 0xDFDF8DA4, 0xA3A35C97, 0xEAEAD57E, 0x353558DA, + 0xEDEDD07A, 0x4343FC17, 0xF8F8CB66, 0xFBFBB194, 0x3737D3A1, 0xFAFA401D, + 0xC2C2683D, 0xB4B4CCF0, 0x32325DDE, 0x9C9C71B3, 0x5656E70B, 0xE3E3DA72, + 0x878760A7, 0x15151B1C, 0xF9F93AEF, 0x6363BFD1, 0x3434A953, 0x9A9A853E, + 0xB1B1428F, 0x7C7CD133, 0x88889B26, 0x3D3DA65F, 0xA1A1D7EC, 0xE4E4DF76, + 0x8181942A, 0x91910149, 0x0F0FFB81, 0xEEEEAA88, 0x161661EE, 0xD7D77321, + 0x9797F5C4, 0xA5A5A81A, 0xFEFE3FEB, 0x6D6DB5D9, 0x7878AEC5, 0xC5C56D39, + 0x1D1DE599, 0x7676A4CD, 0x3E3EDCAD, 0xCBCB6731, 0xB6B6478B, 0xEFEF5B01, + 0x12121E18, 0x6060C523, 0x6A6AB0DD, 0x4D4DF61F, 0xCECEE94E, 0xDEDE7C2D, + 0x55559DF9, 0x7E7E5A48, 0x2121B24F, 0x03037AF2, 0xA0A02665, 0x5E5E198E, + 0x5A5A6678, 0x65654B5C, 0x62624E58, 0xFDFD4519, 0x0606F48D, 0x404086E5, + 0xF2F2BE98, 0x3333AC57, 0x17179067, 0x05058E7F, 0xE8E85E05, 0x4F4F7D64, + 0x89896AAF, 0x10109563, 0x74742FB6, 0x0A0A75FE, 0x5C5C92F5, 0x9B9B74B7, + 0x2D2D333C, 0x3030D6A5, 0x2E2E49CE, 0x494989E9, 0x46467268, 0x77775544, + 0xA8A8D8E0, 0x9696044D, 0x2828BD43, 0xA9A92969, 0xD9D97929, 0x8686912E, + 0xD1D187AC, 0xF4F44A15, 0x8D8D1559, 0xD6D682A8, 0xB9B9BC0A, 0x42420D9E, + 0xF6F6C16E, 0x2F2FB847, 0xDDDD06DF, 0x23233934, 0xCCCC6235, 0xF1F1C46A, + 0xC1C112CF, 0x8585EBDC, 0x8F8F9E22, 0x7171A1C9, 0x9090F0C0, 0xAAAA539B, + 0x0101F189, 0x8B8BE1D4, 0x4E4E8CED, 0x8E8E6FAB, 0xABABA212, 0x6F6F3EA2, + 0xE6E6540D, 0xDBDBF252, 0x92927BBB, 0xB7B7B602, 0x6969CA2F, 0x3939D9A9, + 0xD3D30CD7, 0xA7A72361, 0xA2A2AD1E, 0xC3C399B4, 0x6C6C4450, 0x07070504, + 0x04047FF6, 0x272746C2, 0xACACA716, 0xD0D07625, 0x50501386, 0xDCDCF756, + 0x84841A55, 0xE1E15109, 0x7A7A25BE, 0x1313EF91 }, + + { 0xA9D93939, 0x67901717, 0xB3719C9C, 0xE8D2A6A6, 0x04050707, 0xFD985252, + 0xA3658080, 0x76DFE4E4, 0x9A084545, 0x92024B4B, 0x80A0E0E0, 0x78665A5A, + 0xE4DDAFAF, 0xDDB06A6A, 0xD1BF6363, 0x38362A2A, 0x0D54E6E6, 0xC6432020, + 0x3562CCCC, 0x98BEF2F2, 0x181E1212, 0xF724EBEB, 0xECD7A1A1, 0x6C774141, + 0x43BD2828, 0x7532BCBC, 0x37D47B7B, 0x269B8888, 0xFA700D0D, 0x13F94444, + 0x94B1FBFB, 0x485A7E7E, 0xF27A0303, 0xD0E48C8C, 0x8B47B6B6, 0x303C2424, + 0x84A5E7E7, 0x54416B6B, 0xDF06DDDD, 0x23C56060, 0x1945FDFD, 0x5BA33A3A, + 0x3D68C2C2, 0x59158D8D, 0xF321ECEC, 0xAE316666, 0xA23E6F6F, 0x82165757, + 0x63951010, 0x015BEFEF, 0x834DB8B8, 0x2E918686, 0xD9B56D6D, 0x511F8383, + 0x9B53AAAA, 0x7C635D5D, 0xA63B6868, 0xEB3FFEFE, 0xA5D63030, 0xBE257A7A, + 0x16A7ACAC, 0x0C0F0909, 0xE335F0F0, 0x6123A7A7, 0xC0F09090, 0x8CAFE9E9, + 0x3A809D9D, 0xF5925C5C, 0x73810C0C, 0x2C273131, 0x2576D0D0, 0x0BE75656, + 0xBB7B9292, 0x4EE9CECE, 0x89F10101, 0x6B9F1E1E, 0x53A93434, 0x6AC4F1F1, + 0xB499C3C3, 0xF1975B5B, 0xE1834747, 0xE66B1818, 0xBDC82222, 0x450E9898, + 0xE26E1F1F, 0xF4C9B3B3, 0xB62F7474, 0x66CBF8F8, 0xCCFF9999, 0x95EA1414, + 0x03ED5858, 0x56F7DCDC, 0xD4E18B8B, 0x1C1B1515, 0x1EADA2A2, 0xD70CD3D3, + 0xFB2BE2E2, 0xC31DC8C8, 0x8E195E5E, 0xB5C22C2C, 0xE9894949, 0xCF12C1C1, + 0xBF7E9595, 0xBA207D7D, 0xEA641111, 0x77840B0B, 0x396DC5C5, 0xAF6A8989, + 0x33D17C7C, 0xC9A17171, 0x62CEFFFF, 0x7137BBBB, 0x81FB0F0F, 0x793DB5B5, + 0x0951E1E1, 0xADDC3E3E, 0x242D3F3F, 0xCDA47676, 0xF99D5555, 0xD8EE8282, + 0xE5864040, 0xC5AE7878, 0xB9CD2525, 0x4D049696, 0x44557777, 0x080A0E0E, + 0x86135050, 0xE730F7F7, 0xA1D33737, 0x1D40FAFA, 0xAA346161, 0xED8C4E4E, + 0x06B3B0B0, 0x706C5454, 0xB22A7373, 0xD2523B3B, 0x410B9F9F, 0x7B8B0202, + 0xA088D8D8, 0x114FF3F3, 0x3167CBCB, 0xC2462727, 0x27C06767, 0x90B4FCFC, + 0x20283838, 0xF67F0404, 0x60784848, 0xFF2EE5E5, 0x96074C4C, 0x5C4B6565, + 0xB1C72B2B, 0xAB6F8E8E, 0x9E0D4242, 0x9CBBF5F5, 0x52F2DBDB, 0x1BF34A4A, + 0x5FA63D3D, 0x9359A4A4, 0x0ABCB9B9, 0xEF3AF9F9, 0x91EF1313, 0x85FE0808, + 0x49019191, 0xEE611616, 0x2D7CDEDE, 0x4FB22121, 0x8F42B1B1, 0x3BDB7272, + 0x47B82F2F, 0x8748BFBF, 0x6D2CAEAE, 0x46E3C0C0, 0xD6573C3C, 0x3E859A9A, + 0x6929A9A9, 0x647D4F4F, 0x2A948181, 0xCE492E2E, 0xCB17C6C6, 0x2FCA6969, + 0xFCC3BDBD, 0x975CA3A3, 0x055EE8E8, 0x7AD0EDED, 0xAC87D1D1, 0x7F8E0505, + 0xD5BA6464, 0x1AA8A5A5, 0x4BB72626, 0x0EB9BEBE, 0xA7608787, 0x5AF8D5D5, + 0x28223636, 0x14111B1B, 0x3FDE7575, 0x2979D9D9, 0x88AAEEEE, 0x3C332D2D, + 0x4C5F7979, 0x02B6B7B7, 0xB896CACA, 0xDA583535, 0xB09CC4C4, 0x17FC4343, + 0x551A8484, 0x1FF64D4D, 0x8A1C5959, 0x7D38B2B2, 0x57AC3333, 0xC718CFCF, + 0x8DF40606, 0x74695353, 0xB7749B9B, 0xC4F59797, 0x9F56ADAD, 0x72DAE3E3, + 0x7ED5EAEA, 0x154AF4F4, 0x229E8F8F, 0x12A2ABAB, 0x584E6262, 0x07E85F5F, + 0x99E51D1D, 0x34392323, 0x6EC1F6F6, 0x50446C6C, 0xDE5D3232, 0x68724646, + 0x6526A0A0, 0xBC93CDCD, 0xDB03DADA, 0xF8C6BABA, 0xC8FA9E9E, 0xA882D6D6, + 0x2BCF6E6E, 0x40507070, 0xDCEB8585, 0xFE750A0A, 0x328A9393, 0xA48DDFDF, + 0xCA4C2929, 0x10141C1C, 0x2173D7D7, 0xF0CCB4B4, 0xD309D4D4, 0x5D108A8A, + 0x0FE25151, 0x00000000, 0x6F9A1919, 0x9DE01A1A, 0x368F9494, 0x42E6C7C7, + 0x4AECC9C9, 0x5EFDD2D2, 0xC1AB7F7F, 0xE0D8A8A8 }, + + { 0xBC75BC32, 0xECF3EC21, 0x20C62043, 0xB3F4B3C9, 0xDADBDA03, 0x027B028B, + 0xE2FBE22B, 0x9EC89EFA, 0xC94AC9EC, 0xD4D3D409, 0x18E6186B, 0x1E6B1E9F, + 0x9845980E, 0xB27DB238, 0xA6E8A6D2, 0x264B26B7, 0x3CD63C57, 0x9332938A, + 0x82D882EE, 0x52FD5298, 0x7B377BD4, 0xBB71BB37, 0x5BF15B97, 0x47E14783, + 0x2430243C, 0x510F51E2, 0xBAF8BAC6, 0x4A1B4AF3, 0xBF87BF48, 0x0DFA0D70, + 0xB006B0B3, 0x753F75DE, 0xD25ED2FD, 0x7DBA7D20, 0x66AE6631, 0x3A5B3AA3, + 0x598A591C, 0x00000000, 0xCDBCCD93, 0x1A9D1AE0, 0xAE6DAE2C, 0x7FC17FAB, + 0x2BB12BC7, 0xBE0EBEB9, 0xE080E0A0, 0x8A5D8A10, 0x3BD23B52, 0x64D564BA, + 0xD8A0D888, 0xE784E7A5, 0x5F075FE8, 0x1B141B11, 0x2CB52CC2, 0xFC90FCB4, + 0x312C3127, 0x80A38065, 0x73B2732A, 0x0C730C81, 0x794C795F, 0x6B546B41, + 0x4B924B02, 0x53745369, 0x9436948F, 0x8351831F, 0x2A382A36, 0xC4B0C49C, + 0x22BD22C8, 0xD55AD5F8, 0xBDFCBDC3, 0x48604878, 0xFF62FFCE, 0x4C964C07, + 0x416C4177, 0xC742C7E6, 0xEBF7EB24, 0x1C101C14, 0x5D7C5D63, 0x36283622, + 0x672767C0, 0xE98CE9AF, 0x441344F9, 0x149514EA, 0xF59CF5BB, 0xCFC7CF18, + 0x3F243F2D, 0xC046C0E3, 0x723B72DB, 0x5470546C, 0x29CA294C, 0xF0E3F035, + 0x088508FE, 0xC6CBC617, 0xF311F34F, 0x8CD08CE4, 0xA493A459, 0xCAB8CA96, + 0x68A6683B, 0xB883B84D, 0x38203828, 0xE5FFE52E, 0xAD9FAD56, 0x0B770B84, + 0xC8C3C81D, 0x99CC99FF, 0x580358ED, 0x196F199A, 0x0E080E0A, 0x95BF957E, + 0x70407050, 0xF7E7F730, 0x6E2B6ECF, 0x1FE21F6E, 0xB579B53D, 0x090C090F, + 0x61AA6134, 0x57825716, 0x9F419F0B, 0x9D3A9D80, 0x11EA1164, 0x25B925CD, + 0xAFE4AFDD, 0x459A4508, 0xDFA4DF8D, 0xA397A35C, 0xEA7EEAD5, 0x35DA3558, + 0xED7AEDD0, 0x431743FC, 0xF866F8CB, 0xFB94FBB1, 0x37A137D3, 0xFA1DFA40, + 0xC23DC268, 0xB4F0B4CC, 0x32DE325D, 0x9CB39C71, 0x560B56E7, 0xE372E3DA, + 0x87A78760, 0x151C151B, 0xF9EFF93A, 0x63D163BF, 0x345334A9, 0x9A3E9A85, + 0xB18FB142, 0x7C337CD1, 0x8826889B, 0x3D5F3DA6, 0xA1ECA1D7, 0xE476E4DF, + 0x812A8194, 0x91499101, 0x0F810FFB, 0xEE88EEAA, 0x16EE1661, 0xD721D773, + 0x97C497F5, 0xA51AA5A8, 0xFEEBFE3F, 0x6DD96DB5, 0x78C578AE, 0xC539C56D, + 0x1D991DE5, 0x76CD76A4, 0x3EAD3EDC, 0xCB31CB67, 0xB68BB647, 0xEF01EF5B, + 0x1218121E, 0x602360C5, 0x6ADD6AB0, 0x4D1F4DF6, 0xCE4ECEE9, 0xDE2DDE7C, + 0x55F9559D, 0x7E487E5A, 0x214F21B2, 0x03F2037A, 0xA065A026, 0x5E8E5E19, + 0x5A785A66, 0x655C654B, 0x6258624E, 0xFD19FD45, 0x068D06F4, 0x40E54086, + 0xF298F2BE, 0x335733AC, 0x17671790, 0x057F058E, 0xE805E85E, 0x4F644F7D, + 0x89AF896A, 0x10631095, 0x74B6742F, 0x0AFE0A75, 0x5CF55C92, 0x9BB79B74, + 0x2D3C2D33, 0x30A530D6, 0x2ECE2E49, 0x49E94989, 0x46684672, 0x77447755, + 0xA8E0A8D8, 0x964D9604, 0x284328BD, 0xA969A929, 0xD929D979, 0x862E8691, + 0xD1ACD187, 0xF415F44A, 0x8D598D15, 0xD6A8D682, 0xB90AB9BC, 0x429E420D, + 0xF66EF6C1, 0x2F472FB8, 0xDDDFDD06, 0x23342339, 0xCC35CC62, 0xF16AF1C4, + 0xC1CFC112, 0x85DC85EB, 0x8F228F9E, 0x71C971A1, 0x90C090F0, 0xAA9BAA53, + 0x018901F1, 0x8BD48BE1, 0x4EED4E8C, 0x8EAB8E6F, 0xAB12ABA2, 0x6FA26F3E, + 0xE60DE654, 0xDB52DBF2, 0x92BB927B, 0xB702B7B6, 0x692F69CA, 0x39A939D9, + 0xD3D7D30C, 0xA761A723, 0xA21EA2AD, 0xC3B4C399, 0x6C506C44, 0x07040705, + 0x04F6047F, 0x27C22746, 0xAC16ACA7, 0xD025D076, 0x50865013, 0xDC56DCF7, + 0x8455841A, 0xE109E151, 0x7ABE7A25, 0x139113EF }, + + { 0xD939A9D9, 0x90176790, 0x719CB371, 0xD2A6E8D2, 0x05070405, 0x9852FD98, + 0x6580A365, 0xDFE476DF, 0x08459A08, 0x024B9202, 0xA0E080A0, 0x665A7866, + 0xDDAFE4DD, 0xB06ADDB0, 0xBF63D1BF, 0x362A3836, 0x54E60D54, 0x4320C643, + 0x62CC3562, 0xBEF298BE, 0x1E12181E, 0x24EBF724, 0xD7A1ECD7, 0x77416C77, + 0xBD2843BD, 0x32BC7532, 0xD47B37D4, 0x9B88269B, 0x700DFA70, 0xF94413F9, + 0xB1FB94B1, 0x5A7E485A, 0x7A03F27A, 0xE48CD0E4, 0x47B68B47, 0x3C24303C, + 0xA5E784A5, 0x416B5441, 0x06DDDF06, 0xC56023C5, 0x45FD1945, 0xA33A5BA3, + 0x68C23D68, 0x158D5915, 0x21ECF321, 0x3166AE31, 0x3E6FA23E, 0x16578216, + 0x95106395, 0x5BEF015B, 0x4DB8834D, 0x91862E91, 0xB56DD9B5, 0x1F83511F, + 0x53AA9B53, 0x635D7C63, 0x3B68A63B, 0x3FFEEB3F, 0xD630A5D6, 0x257ABE25, + 0xA7AC16A7, 0x0F090C0F, 0x35F0E335, 0x23A76123, 0xF090C0F0, 0xAFE98CAF, + 0x809D3A80, 0x925CF592, 0x810C7381, 0x27312C27, 0x76D02576, 0xE7560BE7, + 0x7B92BB7B, 0xE9CE4EE9, 0xF10189F1, 0x9F1E6B9F, 0xA93453A9, 0xC4F16AC4, + 0x99C3B499, 0x975BF197, 0x8347E183, 0x6B18E66B, 0xC822BDC8, 0x0E98450E, + 0x6E1FE26E, 0xC9B3F4C9, 0x2F74B62F, 0xCBF866CB, 0xFF99CCFF, 0xEA1495EA, + 0xED5803ED, 0xF7DC56F7, 0xE18BD4E1, 0x1B151C1B, 0xADA21EAD, 0x0CD3D70C, + 0x2BE2FB2B, 0x1DC8C31D, 0x195E8E19, 0xC22CB5C2, 0x8949E989, 0x12C1CF12, + 0x7E95BF7E, 0x207DBA20, 0x6411EA64, 0x840B7784, 0x6DC5396D, 0x6A89AF6A, + 0xD17C33D1, 0xA171C9A1, 0xCEFF62CE, 0x37BB7137, 0xFB0F81FB, 0x3DB5793D, + 0x51E10951, 0xDC3EADDC, 0x2D3F242D, 0xA476CDA4, 0x9D55F99D, 0xEE82D8EE, + 0x8640E586, 0xAE78C5AE, 0xCD25B9CD, 0x04964D04, 0x55774455, 0x0A0E080A, + 0x13508613, 0x30F7E730, 0xD337A1D3, 0x40FA1D40, 0x3461AA34, 0x8C4EED8C, + 0xB3B006B3, 0x6C54706C, 0x2A73B22A, 0x523BD252, 0x0B9F410B, 0x8B027B8B, + 0x88D8A088, 0x4FF3114F, 0x67CB3167, 0x4627C246, 0xC06727C0, 0xB4FC90B4, + 0x28382028, 0x7F04F67F, 0x78486078, 0x2EE5FF2E, 0x074C9607, 0x4B655C4B, + 0xC72BB1C7, 0x6F8EAB6F, 0x0D429E0D, 0xBBF59CBB, 0xF2DB52F2, 0xF34A1BF3, + 0xA63D5FA6, 0x59A49359, 0xBCB90ABC, 0x3AF9EF3A, 0xEF1391EF, 0xFE0885FE, + 0x01914901, 0x6116EE61, 0x7CDE2D7C, 0xB2214FB2, 0x42B18F42, 0xDB723BDB, + 0xB82F47B8, 0x48BF8748, 0x2CAE6D2C, 0xE3C046E3, 0x573CD657, 0x859A3E85, + 0x29A96929, 0x7D4F647D, 0x94812A94, 0x492ECE49, 0x17C6CB17, 0xCA692FCA, + 0xC3BDFCC3, 0x5CA3975C, 0x5EE8055E, 0xD0ED7AD0, 0x87D1AC87, 0x8E057F8E, + 0xBA64D5BA, 0xA8A51AA8, 0xB7264BB7, 0xB9BE0EB9, 0x6087A760, 0xF8D55AF8, + 0x22362822, 0x111B1411, 0xDE753FDE, 0x79D92979, 0xAAEE88AA, 0x332D3C33, + 0x5F794C5F, 0xB6B702B6, 0x96CAB896, 0x5835DA58, 0x9CC4B09C, 0xFC4317FC, + 0x1A84551A, 0xF64D1FF6, 0x1C598A1C, 0x38B27D38, 0xAC3357AC, 0x18CFC718, + 0xF4068DF4, 0x69537469, 0x749BB774, 0xF597C4F5, 0x56AD9F56, 0xDAE372DA, + 0xD5EA7ED5, 0x4AF4154A, 0x9E8F229E, 0xA2AB12A2, 0x4E62584E, 0xE85F07E8, + 0xE51D99E5, 0x39233439, 0xC1F66EC1, 0x446C5044, 0x5D32DE5D, 0x72466872, + 0x26A06526, 0x93CDBC93, 0x03DADB03, 0xC6BAF8C6, 0xFA9EC8FA, 0x82D6A882, + 0xCF6E2BCF, 0x50704050, 0xEB85DCEB, 0x750AFE75, 0x8A93328A, 0x8DDFA48D, + 0x4C29CA4C, 0x141C1014, 0x73D72173, 0xCCB4F0CC, 0x09D4D309, 0x108A5D10, + 0xE2510FE2, 0x00000000, 0x9A196F9A, 0xE01A9DE0, 0x8F94368F, 0xE6C742E6, + 0xECC94AEC, 0xFDD25EFD, 0xAB7FC1AB, 0xD8A8E0D8 } +}; + +#define mds(n,x) m_tab[n][x] + +u32 h_fun (u32 *sk, u32 *lk, const u32 x, const u32 *key) +{ + u32 b0, b1, b2, b3; + + b0 = extract_byte (x, 0); + b1 = extract_byte (x, 1); + b2 = extract_byte (x, 2); + b3 = extract_byte (x, 3); + + b0 = q (1, b0) ^ extract_byte (key[3], 0); + b1 = q (0, b1) ^ extract_byte (key[3], 1); + b2 = q (0, b2) ^ extract_byte (key[3], 2); + b3 = q (1, b3) ^ extract_byte (key[3], 3); + + b0 = q (1, b0) ^ extract_byte (key[2], 0); + b1 = q (1, b1) ^ extract_byte (key[2], 1); + b2 = q (0, b2) ^ extract_byte (key[2], 2); + b3 = q (0, b3) ^ extract_byte (key[2], 3); + + b0 = q (0, (q (0, b0) ^ extract_byte (key[1], 0))) ^ extract_byte (key[0], 0); + b1 = q (0, (q (1, b1) ^ extract_byte (key[1], 1))) ^ extract_byte (key[0], 1); + b2 = q (1, (q (0, b2) ^ extract_byte (key[1], 2))) ^ extract_byte (key[0], 2); + b3 = q (1, (q (1, b3) ^ extract_byte (key[1], 3))) ^ extract_byte (key[0], 3); + + return mds (0, b0) ^ mds (1, b1) ^ mds (2, b2) ^ mds (3, b3); +} + +#define q40(x,k) q (0, q (0, q (1, q (1, x) ^ extract_byte (k[3], 0)) ^ extract_byte (k[2], 0)) ^ extract_byte (k[1], 0)) ^ extract_byte (k[0], 0) +#define q41(x,k) q (0, q (1, q (1, q (0, x) ^ extract_byte (k[3], 1)) ^ extract_byte (k[2], 1)) ^ extract_byte (k[1], 1)) ^ extract_byte (k[0], 1) +#define q42(x,k) q (1, q (0, q (0, q (0, x) ^ extract_byte (k[3], 2)) ^ extract_byte (k[2], 2)) ^ extract_byte (k[1], 2)) ^ extract_byte (k[0], 2) +#define q43(x,k) q (1, q (1, q (0, q (1, x) ^ extract_byte (k[3], 3)) ^ extract_byte (k[2], 3)) ^ extract_byte (k[1], 3)) ^ extract_byte (k[0], 3) + +#define g1_fun(x) \ + (mds (0, q40 (extract_byte (x, 3), sk)) ^ \ + mds (1, q41 (extract_byte (x, 0), sk)) ^ \ + mds (2, q42 (extract_byte (x, 1), sk)) ^ \ + mds (3, q43 (extract_byte (x, 2), sk))) + +#define g0_fun(x) \ + (mds (0, q40 (extract_byte (x, 0), sk)) ^ \ + mds (1, q41 (extract_byte (x, 1), sk)) ^ \ + mds (2, q42 (extract_byte (x, 2), sk)) ^ \ + mds (3, q43 (extract_byte (x, 3), sk))) + +u32 mds_rem (u32 p0, u32 p1) +{ + #define G_MOD 0x14d + + for (int i = 0; i < 8; i++) + { + u32 t = p1 >> 24; + + p1 = (p1 << 8) | (p0 >> 24); + + p0 <<= 8; + + u32 u = (t << 1); + + if (t & 0x80) u ^= G_MOD; + + p1 ^= t ^ (u << 16); + + u ^= (t >> 1); + + if (t & 0x01) u ^= G_MOD >> 1; + + p1 ^= (u << 24) | (u << 8); + } + + return p1; +} + +void twofish256_set_key (u32 *sk, u32 *lk, const u32 *ukey) +{ + u32 me_key[4]; + + me_key[0] = ukey[0]; + me_key[1] = ukey[2]; + me_key[2] = ukey[4]; + me_key[3] = ukey[6]; + + u32 mo_key[4]; + + mo_key[0] = ukey[1]; + mo_key[1] = ukey[3]; + mo_key[2] = ukey[5]; + mo_key[3] = ukey[7]; + + sk[3] = mds_rem (me_key[0], mo_key[0]); + sk[2] = mds_rem (me_key[1], mo_key[1]); + sk[1] = mds_rem (me_key[2], mo_key[2]); + sk[0] = mds_rem (me_key[3], mo_key[3]); + + for (int i = 0; i < 40; i += 2) + { + u32 a = 0x01010101 * i; + u32 b = 0x01010101 + a; + + a = h_fun (sk, lk, a, me_key); + b = h_fun (sk, lk, b, mo_key); + + b = rotl32 (b, 8); + + lk[i + 0] = a + b; + lk[i + 1] = rotl32 (a + 2 * b, 9); + } +} + +#define f_rnd(i) \ +{ \ + u32 t0 = g0_fun (data[0]); \ + u32 t1 = g1_fun (data[1]); \ + data[2] = rotr32 (data[2] ^ (t0 + t1 + lk[4 * (i) + 8]), 1); \ + data[3] = rotl32 (data[3], 1) ^ (t0 + 2 * t1 + lk[4 * (i) + 9]); \ + u32 t2 = g0_fun (data[2]); \ + u32 t3 = g1_fun (data[3]); \ + data[0] = rotr32 (data[0] ^ (t2 + t3 + lk[4 * (i) + 10]), 1); \ + data[1] = rotl32 (data[1], 1) ^ (t2 + 2 * t3 + lk[4 * (i) + 11]); \ +} + +void twofish256_encrypt (const u32 *sk, const u32 *lk, const u32 *in, u32 *out) +{ + u32 data[4]; + + data[0] = in[0] ^ lk[0]; + data[1] = in[1] ^ lk[1]; + data[2] = in[2] ^ lk[2]; + data[3] = in[3] ^ lk[3]; + + f_rnd (0); + f_rnd (1); + f_rnd (2); + f_rnd (3); + f_rnd (4); + f_rnd (5); + f_rnd (6); + f_rnd (7); + + out[0] = data[2] ^ lk[4]; + out[1] = data[3] ^ lk[5]; + out[2] = data[0] ^ lk[6]; + out[3] = data[1] ^ lk[7]; +} + +#define i_rnd(i) \ +{ \ + u32 t0 = g0_fun (data[0]); \ + u32 t1 = g1_fun (data[1]); \ + data[2] = rotl32 (data[2], 1) ^ (t0 + t1 + lk[4 * (i) + 10]); \ + data[3] = rotr32 (data[3] ^ (t0 + 2 * t1 + lk[4 * (i) + 11]), 1); \ + u32 t2 = g0_fun (data[2]); \ + u32 t3 = g1_fun (data[3]); \ + data[0] = rotl32 (data[0], 1) ^ (t2 + t3 + lk[4 * (i) + 8]); \ + data[1] = rotr32 (data[1] ^ (t2 + 2 * t3 + lk[4 * (i) + 9]), 1); \ +} + +void twofish256_decrypt (const u32 *sk, const u32 *lk, const u32 *in, u32 *out) +{ + u32 data[4]; + + data[0] = in[0] ^ lk[4]; + data[1] = in[1] ^ lk[5]; + data[2] = in[2] ^ lk[6]; + data[3] = in[3] ^ lk[7]; + + i_rnd (7); + i_rnd (6); + i_rnd (5); + i_rnd (4); + i_rnd (3); + i_rnd (2); + i_rnd (1); + i_rnd (0); + + out[0] = data[2] ^ lk[0]; + out[1] = data[3] ^ lk[1]; + out[2] = data[0] ^ lk[2]; + out[3] = data[1] ^ lk[3]; +} + +void twofish256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out) +{ + u32 T[4] = { 0 }; + u32 Z[4] = { 0 }; + + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; + + u32 sk[4]; u32 lk[40]; + + twofish256_set_key (sk, lk, ukey2); + twofish256_encrypt (sk, lk, Z, T); + + out[0] ^= T[0]; + out[1] ^= T[1]; + out[2] ^= T[2]; + out[3] ^= T[3]; + + twofish256_set_key (sk, lk, ukey1); + twofish256_decrypt (sk, lk, out, out); + + out[0] ^= T[0]; + out[1] ^= T[1]; + out[2] ^= T[2]; + out[3] ^= T[3]; +} diff --git a/OpenCL/inc_common.cl b/OpenCL/inc_common.cl new file mode 100644 index 0000000..ca811e5 --- /dev/null +++ b/OpenCL/inc_common.cl @@ -0,0 +1,9324 @@ +/** + * Author......: Jens Steube + * License.....: MIT + */ + +/** + * pure scalar functions + */ + +inline int hash_comp (const u32 d1[4], __global u32 *d2) +{ + if (d1[3] > d2[DGST_R3]) return ( 1); + if (d1[3] < d2[DGST_R3]) return (-1); + if (d1[2] > d2[DGST_R2]) return ( 1); + if (d1[2] < d2[DGST_R2]) return (-1); + if (d1[1] > d2[DGST_R1]) return ( 1); + if (d1[1] < d2[DGST_R1]) return (-1); + if (d1[0] > d2[DGST_R0]) return ( 1); + if (d1[0] < d2[DGST_R0]) return (-1); + + return (0); +} + +inline int find_hash (const u32 digest[4], const u32 digests_cnt, __global digest_t *digests_buf) +{ + for (u32 l = 0, r = digests_cnt; r; r >>= 1) + { + const u32 m = r >> 1; + + const u32 c = l + m; + + const int cmp = hash_comp (digest, digests_buf[c].digest_buf); + + if (cmp > 0) + { + l += m + 1; + + r--; + } + + if (cmp == 0) return (c); + } + + return (-1); +} + +inline u32 check_bitmap (__global u32 *bitmap, const u32 bitmap_mask, const u32 bitmap_shift, const u32 digest) +{ + return (bitmap[(digest >> bitmap_shift) & bitmap_mask] & (1 << (digest & 0x1f))); +} + +inline u32 check (const u32 digest[2], __global u32 *bitmap_s1_a, __global u32 *bitmap_s1_b, __global u32 *bitmap_s1_c, __global u32 *bitmap_s1_d, __global u32 *bitmap_s2_a, __global u32 *bitmap_s2_b, __global u32 *bitmap_s2_c, __global u32 *bitmap_s2_d, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2) +{ + if (check_bitmap (bitmap_s1_a, bitmap_mask, bitmap_shift1, digest[0]) == 0) return (0); + if (check_bitmap (bitmap_s1_b, bitmap_mask, bitmap_shift1, digest[1]) == 0) return (0); + if (check_bitmap (bitmap_s1_c, bitmap_mask, bitmap_shift1, digest[2]) == 0) return (0); + if (check_bitmap (bitmap_s1_d, bitmap_mask, bitmap_shift1, digest[3]) == 0) return (0); + + if (check_bitmap (bitmap_s2_a, bitmap_mask, bitmap_shift2, digest[0]) == 0) return (0); + if (check_bitmap (bitmap_s2_b, bitmap_mask, bitmap_shift2, digest[1]) == 0) return (0); + if (check_bitmap (bitmap_s2_c, bitmap_mask, bitmap_shift2, digest[2]) == 0) return (0); + if (check_bitmap (bitmap_s2_d, bitmap_mask, bitmap_shift2, digest[3]) == 0) return (0); + + return (1); +} + +inline void mark_hash (__global plain_t *plains_buf, __global u32 *d_result, const int salt_pos, const int digest_pos, const int hash_pos, const u32 gid, const u32 il_pos) +{ + const u32 idx = atomic_inc (d_result); + + plains_buf[idx].salt_pos = salt_pos; + plains_buf[idx].digest_pos = digest_pos; // relative + plains_buf[idx].hash_pos = hash_pos; // absolute + plains_buf[idx].gidvid = gid; + plains_buf[idx].il_pos = il_pos; +} + +/** + * vector functions + */ + +inline void truncate_block (u32x w[4], const u32 len) +{ + switch (len) + { + case 0: w[0] &= 0; + w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 1: w[0] &= 0x000000FF; + w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 2: w[0] &= 0x0000FFFF; + w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 3: w[0] &= 0x00FFFFFF; + w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 4: w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 5: w[1] &= 0x000000FF; + w[2] &= 0; + w[3] &= 0; + break; + case 6: w[1] &= 0x0000FFFF; + w[2] &= 0; + w[3] &= 0; + break; + case 7: w[1] &= 0x00FFFFFF; + w[2] &= 0; + w[3] &= 0; + break; + case 8: w[2] &= 0; + w[3] &= 0; + break; + case 9: w[2] &= 0x000000FF; + w[3] &= 0; + break; + case 10: w[2] &= 0x0000FFFF; + w[3] &= 0; + break; + case 11: w[2] &= 0x00FFFFFF; + w[3] &= 0; + break; + case 12: w[3] &= 0; + break; + case 13: w[3] &= 0x000000FF; + break; + case 14: w[3] &= 0x0000FFFF; + break; + case 15: w[3] &= 0x00FFFFFF; + break; + } +} + +inline void make_unicode (const u32x in[4], u32x out1[4], u32x out2[4]) +{ + #ifdef IS_NV + out2[3] = __byte_perm (in[3], 0, 0x7372); + out2[2] = __byte_perm (in[3], 0, 0x7170); + out2[1] = __byte_perm (in[2], 0, 0x7372); + out2[0] = __byte_perm (in[2], 0, 0x7170); + out1[3] = __byte_perm (in[1], 0, 0x7372); + out1[2] = __byte_perm (in[1], 0, 0x7170); + out1[1] = __byte_perm (in[0], 0, 0x7372); + out1[0] = __byte_perm (in[0], 0, 0x7170); + #endif + + #if defined IS_AMD || defined IS_GENERIC + out2[3] = ((in[3] >> 8) & 0x00FF0000) | ((in[3] >> 16) & 0x000000FF); + out2[2] = ((in[3] << 8) & 0x00FF0000) | ((in[3] >> 0) & 0x000000FF); + out2[1] = ((in[2] >> 8) & 0x00FF0000) | ((in[2] >> 16) & 0x000000FF); + out2[0] = ((in[2] << 8) & 0x00FF0000) | ((in[2] >> 0) & 0x000000FF); + out1[3] = ((in[1] >> 8) & 0x00FF0000) | ((in[1] >> 16) & 0x000000FF); + out1[2] = ((in[1] << 8) & 0x00FF0000) | ((in[1] >> 0) & 0x000000FF); + out1[1] = ((in[0] >> 8) & 0x00FF0000) | ((in[0] >> 16) & 0x000000FF); + out1[0] = ((in[0] << 8) & 0x00FF0000) | ((in[0] >> 0) & 0x000000FF); + #endif +} + +inline void undo_unicode (const u32x in1[4], const u32x in2[4], u32x out[4]) +{ + #ifdef IS_NV + out[0] = __byte_perm (in1[0], in1[1], 0x6420); + out[1] = __byte_perm (in1[2], in1[3], 0x6420); + out[2] = __byte_perm (in2[0], in2[1], 0x6420); + out[3] = __byte_perm (in2[2], in2[3], 0x6420); + #endif + + #if defined IS_AMD || defined IS_GENERIC + out[0] = ((in1[0] & 0x000000ff) >> 0) | ((in1[0] & 0x00ff0000) >> 8) + | ((in1[1] & 0x000000ff) << 16) | ((in1[1] & 0x00ff0000) << 8); + out[1] = ((in1[2] & 0x000000ff) >> 0) | ((in1[2] & 0x00ff0000) >> 8) + | ((in1[3] & 0x000000ff) << 16) | ((in1[3] & 0x00ff0000) << 8); + out[2] = ((in2[0] & 0x000000ff) >> 0) | ((in2[0] & 0x00ff0000) >> 8) + | ((in2[1] & 0x000000ff) << 16) | ((in2[1] & 0x00ff0000) << 8); + out[3] = ((in2[2] & 0x000000ff) >> 0) | ((in2[2] & 0x00ff0000) >> 8) + | ((in2[3] & 0x000000ff) << 16) | ((in2[3] & 0x00ff0000) << 8); + #endif +} + +inline void append_0x01_1x4 (u32x w0[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x01; + break; + + case 1: + w0[0] = w0[0] | 0x0100; + break; + + case 2: + w0[0] = w0[0] | 0x010000; + break; + + case 3: + w0[0] = w0[0] | 0x01000000; + break; + + case 4: + w0[1] = 0x01; + break; + + case 5: + w0[1] = w0[1] | 0x0100; + break; + + case 6: + w0[1] = w0[1] | 0x010000; + break; + + case 7: + w0[1] = w0[1] | 0x01000000; + break; + + case 8: + w0[2] = 0x01; + break; + + case 9: + w0[2] = w0[2] | 0x0100; + break; + + case 10: + w0[2] = w0[2] | 0x010000; + break; + + case 11: + w0[2] = w0[2] | 0x01000000; + break; + + case 12: + w0[3] = 0x01; + break; + + case 13: + w0[3] = w0[3] | 0x0100; + break; + + case 14: + w0[3] = w0[3] | 0x010000; + break; + + case 15: + w0[3] = w0[3] | 0x01000000; + break; + } +} + +inline void append_0x01_2x4 (u32x w0[4], u32x w1[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x01; + break; + + case 1: + w0[0] = w0[0] | 0x0100; + break; + + case 2: + w0[0] = w0[0] | 0x010000; + break; + + case 3: + w0[0] = w0[0] | 0x01000000; + break; + + case 4: + w0[1] = 0x01; + break; + + case 5: + w0[1] = w0[1] | 0x0100; + break; + + case 6: + w0[1] = w0[1] | 0x010000; + break; + + case 7: + w0[1] = w0[1] | 0x01000000; + break; + + case 8: + w0[2] = 0x01; + break; + + case 9: + w0[2] = w0[2] | 0x0100; + break; + + case 10: + w0[2] = w0[2] | 0x010000; + break; + + case 11: + w0[2] = w0[2] | 0x01000000; + break; + + case 12: + w0[3] = 0x01; + break; + + case 13: + w0[3] = w0[3] | 0x0100; + break; + + case 14: + w0[3] = w0[3] | 0x010000; + break; + + case 15: + w0[3] = w0[3] | 0x01000000; + break; + + case 16: + w1[0] = 0x01; + break; + + case 17: + w1[0] = w1[0] | 0x0100; + break; + + case 18: + w1[0] = w1[0] | 0x010000; + break; + + case 19: + w1[0] = w1[0] | 0x01000000; + break; + + case 20: + w1[1] = 0x01; + break; + + case 21: + w1[1] = w1[1] | 0x0100; + break; + + case 22: + w1[1] = w1[1] | 0x010000; + break; + + case 23: + w1[1] = w1[1] | 0x01000000; + break; + + case 24: + w1[2] = 0x01; + break; + + case 25: + w1[2] = w1[2] | 0x0100; + break; + + case 26: + w1[2] = w1[2] | 0x010000; + break; + + case 27: + w1[2] = w1[2] | 0x01000000; + break; + + case 28: + w1[3] = 0x01; + break; + + case 29: + w1[3] = w1[3] | 0x0100; + break; + + case 30: + w1[3] = w1[3] | 0x010000; + break; + + case 31: + w1[3] = w1[3] | 0x01000000; + break; + } +} + +inline void append_0x01_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x01; + break; + + case 1: + w0[0] = w0[0] | 0x0100; + break; + + case 2: + w0[0] = w0[0] | 0x010000; + break; + + case 3: + w0[0] = w0[0] | 0x01000000; + break; + + case 4: + w0[1] = 0x01; + break; + + case 5: + w0[1] = w0[1] | 0x0100; + break; + + case 6: + w0[1] = w0[1] | 0x010000; + break; + + case 7: + w0[1] = w0[1] | 0x01000000; + break; + + case 8: + w0[2] = 0x01; + break; + + case 9: + w0[2] = w0[2] | 0x0100; + break; + + case 10: + w0[2] = w0[2] | 0x010000; + break; + + case 11: + w0[2] = w0[2] | 0x01000000; + break; + + case 12: + w0[3] = 0x01; + break; + + case 13: + w0[3] = w0[3] | 0x0100; + break; + + case 14: + w0[3] = w0[3] | 0x010000; + break; + + case 15: + w0[3] = w0[3] | 0x01000000; + break; + + case 16: + w1[0] = 0x01; + break; + + case 17: + w1[0] = w1[0] | 0x0100; + break; + + case 18: + w1[0] = w1[0] | 0x010000; + break; + + case 19: + w1[0] = w1[0] | 0x01000000; + break; + + case 20: + w1[1] = 0x01; + break; + + case 21: + w1[1] = w1[1] | 0x0100; + break; + + case 22: + w1[1] = w1[1] | 0x010000; + break; + + case 23: + w1[1] = w1[1] | 0x01000000; + break; + + case 24: + w1[2] = 0x01; + break; + + case 25: + w1[2] = w1[2] | 0x0100; + break; + + case 26: + w1[2] = w1[2] | 0x010000; + break; + + case 27: + w1[2] = w1[2] | 0x01000000; + break; + + case 28: + w1[3] = 0x01; + break; + + case 29: + w1[3] = w1[3] | 0x0100; + break; + + case 30: + w1[3] = w1[3] | 0x010000; + break; + + case 31: + w1[3] = w1[3] | 0x01000000; + break; + + case 32: + w2[0] = 0x01; + break; + + case 33: + w2[0] = w2[0] | 0x0100; + break; + + case 34: + w2[0] = w2[0] | 0x010000; + break; + + case 35: + w2[0] = w2[0] | 0x01000000; + break; + + case 36: + w2[1] = 0x01; + break; + + case 37: + w2[1] = w2[1] | 0x0100; + break; + + case 38: + w2[1] = w2[1] | 0x010000; + break; + + case 39: + w2[1] = w2[1] | 0x01000000; + break; + + case 40: + w2[2] = 0x01; + break; + + case 41: + w2[2] = w2[2] | 0x0100; + break; + + case 42: + w2[2] = w2[2] | 0x010000; + break; + + case 43: + w2[2] = w2[2] | 0x01000000; + break; + + case 44: + w2[3] = 0x01; + break; + + case 45: + w2[3] = w2[3] | 0x0100; + break; + + case 46: + w2[3] = w2[3] | 0x010000; + break; + + case 47: + w2[3] = w2[3] | 0x01000000; + break; + } +} + +inline void append_0x01_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x01; + break; + + case 1: + w0[0] = w0[0] | 0x0100; + break; + + case 2: + w0[0] = w0[0] | 0x010000; + break; + + case 3: + w0[0] = w0[0] | 0x01000000; + break; + + case 4: + w0[1] = 0x01; + break; + + case 5: + w0[1] = w0[1] | 0x0100; + break; + + case 6: + w0[1] = w0[1] | 0x010000; + break; + + case 7: + w0[1] = w0[1] | 0x01000000; + break; + + case 8: + w0[2] = 0x01; + break; + + case 9: + w0[2] = w0[2] | 0x0100; + break; + + case 10: + w0[2] = w0[2] | 0x010000; + break; + + case 11: + w0[2] = w0[2] | 0x01000000; + break; + + case 12: + w0[3] = 0x01; + break; + + case 13: + w0[3] = w0[3] | 0x0100; + break; + + case 14: + w0[3] = w0[3] | 0x010000; + break; + + case 15: + w0[3] = w0[3] | 0x01000000; + break; + + case 16: + w1[0] = 0x01; + break; + + case 17: + w1[0] = w1[0] | 0x0100; + break; + + case 18: + w1[0] = w1[0] | 0x010000; + break; + + case 19: + w1[0] = w1[0] | 0x01000000; + break; + + case 20: + w1[1] = 0x01; + break; + + case 21: + w1[1] = w1[1] | 0x0100; + break; + + case 22: + w1[1] = w1[1] | 0x010000; + break; + + case 23: + w1[1] = w1[1] | 0x01000000; + break; + + case 24: + w1[2] = 0x01; + break; + + case 25: + w1[2] = w1[2] | 0x0100; + break; + + case 26: + w1[2] = w1[2] | 0x010000; + break; + + case 27: + w1[2] = w1[2] | 0x01000000; + break; + + case 28: + w1[3] = 0x01; + break; + + case 29: + w1[3] = w1[3] | 0x0100; + break; + + case 30: + w1[3] = w1[3] | 0x010000; + break; + + case 31: + w1[3] = w1[3] | 0x01000000; + break; + + case 32: + w2[0] = 0x01; + break; + + case 33: + w2[0] = w2[0] | 0x0100; + break; + + case 34: + w2[0] = w2[0] | 0x010000; + break; + + case 35: + w2[0] = w2[0] | 0x01000000; + break; + + case 36: + w2[1] = 0x01; + break; + + case 37: + w2[1] = w2[1] | 0x0100; + break; + + case 38: + w2[1] = w2[1] | 0x010000; + break; + + case 39: + w2[1] = w2[1] | 0x01000000; + break; + + case 40: + w2[2] = 0x01; + break; + + case 41: + w2[2] = w2[2] | 0x0100; + break; + + case 42: + w2[2] = w2[2] | 0x010000; + break; + + case 43: + w2[2] = w2[2] | 0x01000000; + break; + + case 44: + w2[3] = 0x01; + break; + + case 45: + w2[3] = w2[3] | 0x0100; + break; + + case 46: + w2[3] = w2[3] | 0x010000; + break; + + case 47: + w2[3] = w2[3] | 0x01000000; + break; + + case 48: + w3[0] = 0x01; + break; + + case 49: + w3[0] = w3[0] | 0x0100; + break; + + case 50: + w3[0] = w3[0] | 0x010000; + break; + + case 51: + w3[0] = w3[0] | 0x01000000; + break; + + case 52: + w3[1] = 0x01; + break; + + case 53: + w3[1] = w3[1] | 0x0100; + break; + + case 54: + w3[1] = w3[1] | 0x010000; + break; + + case 55: + w3[1] = w3[1] | 0x01000000; + break; + + case 56: + w3[2] = 0x01; + break; + + case 57: + w3[2] = w3[2] | 0x0100; + break; + + case 58: + w3[2] = w3[2] | 0x010000; + break; + + case 59: + w3[2] = w3[2] | 0x01000000; + break; + + case 60: + w3[3] = 0x01; + break; + + case 61: + w3[3] = w3[3] | 0x0100; + break; + + case 62: + w3[3] = w3[3] | 0x010000; + break; + + case 63: + w3[3] = w3[3] | 0x01000000; + break; + } +} + +inline void append_0x01_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x01; + break; + + case 1: + w0[0] = w0[0] | 0x0100; + break; + + case 2: + w0[0] = w0[0] | 0x010000; + break; + + case 3: + w0[0] = w0[0] | 0x01000000; + break; + + case 4: + w0[1] = 0x01; + break; + + case 5: + w0[1] = w0[1] | 0x0100; + break; + + case 6: + w0[1] = w0[1] | 0x010000; + break; + + case 7: + w0[1] = w0[1] | 0x01000000; + break; + + case 8: + w0[2] = 0x01; + break; + + case 9: + w0[2] = w0[2] | 0x0100; + break; + + case 10: + w0[2] = w0[2] | 0x010000; + break; + + case 11: + w0[2] = w0[2] | 0x01000000; + break; + + case 12: + w0[3] = 0x01; + break; + + case 13: + w0[3] = w0[3] | 0x0100; + break; + + case 14: + w0[3] = w0[3] | 0x010000; + break; + + case 15: + w0[3] = w0[3] | 0x01000000; + break; + + case 16: + w1[0] = 0x01; + break; + + case 17: + w1[0] = w1[0] | 0x0100; + break; + + case 18: + w1[0] = w1[0] | 0x010000; + break; + + case 19: + w1[0] = w1[0] | 0x01000000; + break; + + case 20: + w1[1] = 0x01; + break; + + case 21: + w1[1] = w1[1] | 0x0100; + break; + + case 22: + w1[1] = w1[1] | 0x010000; + break; + + case 23: + w1[1] = w1[1] | 0x01000000; + break; + + case 24: + w1[2] = 0x01; + break; + + case 25: + w1[2] = w1[2] | 0x0100; + break; + + case 26: + w1[2] = w1[2] | 0x010000; + break; + + case 27: + w1[2] = w1[2] | 0x01000000; + break; + + case 28: + w1[3] = 0x01; + break; + + case 29: + w1[3] = w1[3] | 0x0100; + break; + + case 30: + w1[3] = w1[3] | 0x010000; + break; + + case 31: + w1[3] = w1[3] | 0x01000000; + break; + + case 32: + w2[0] = 0x01; + break; + + case 33: + w2[0] = w2[0] | 0x0100; + break; + + case 34: + w2[0] = w2[0] | 0x010000; + break; + + case 35: + w2[0] = w2[0] | 0x01000000; + break; + + case 36: + w2[1] = 0x01; + break; + + case 37: + w2[1] = w2[1] | 0x0100; + break; + + case 38: + w2[1] = w2[1] | 0x010000; + break; + + case 39: + w2[1] = w2[1] | 0x01000000; + break; + + case 40: + w2[2] = 0x01; + break; + + case 41: + w2[2] = w2[2] | 0x0100; + break; + + case 42: + w2[2] = w2[2] | 0x010000; + break; + + case 43: + w2[2] = w2[2] | 0x01000000; + break; + + case 44: + w2[3] = 0x01; + break; + + case 45: + w2[3] = w2[3] | 0x0100; + break; + + case 46: + w2[3] = w2[3] | 0x010000; + break; + + case 47: + w2[3] = w2[3] | 0x01000000; + break; + + case 48: + w3[0] = 0x01; + break; + + case 49: + w3[0] = w3[0] | 0x0100; + break; + + case 50: + w3[0] = w3[0] | 0x010000; + break; + + case 51: + w3[0] = w3[0] | 0x01000000; + break; + + case 52: + w3[1] = 0x01; + break; + + case 53: + w3[1] = w3[1] | 0x0100; + break; + + case 54: + w3[1] = w3[1] | 0x010000; + break; + + case 55: + w3[1] = w3[1] | 0x01000000; + break; + + case 56: + w3[2] = 0x01; + break; + + case 57: + w3[2] = w3[2] | 0x0100; + break; + + case 58: + w3[2] = w3[2] | 0x010000; + break; + + case 59: + w3[2] = w3[2] | 0x01000000; + break; + + case 60: + w3[3] = 0x01; + break; + + case 61: + w3[3] = w3[3] | 0x0100; + break; + + case 62: + w3[3] = w3[3] | 0x010000; + break; + + case 63: + w3[3] = w3[3] | 0x01000000; + break; + + case 64: + w4[0] = 0x01; + break; + + case 65: + w4[0] = w4[0] | 0x0100; + break; + + case 66: + w4[0] = w4[0] | 0x010000; + break; + + case 67: + w4[0] = w4[0] | 0x01000000; + break; + + case 68: + w4[1] = 0x01; + break; + + case 69: + w4[1] = w4[1] | 0x0100; + break; + + case 70: + w4[1] = w4[1] | 0x010000; + break; + + case 71: + w4[1] = w4[1] | 0x01000000; + break; + + case 72: + w4[2] = 0x01; + break; + + case 73: + w4[2] = w4[2] | 0x0100; + break; + + case 74: + w4[2] = w4[2] | 0x010000; + break; + + case 75: + w4[2] = w4[2] | 0x01000000; + break; + + case 76: + w4[3] = 0x01; + break; + + case 77: + w4[3] = w4[3] | 0x0100; + break; + + case 78: + w4[3] = w4[3] | 0x010000; + break; + + case 79: + w4[3] = w4[3] | 0x01000000; + break; + + case 80: + w5[0] = 0x01; + break; + + case 81: + w5[0] = w5[0] | 0x0100; + break; + + case 82: + w5[0] = w5[0] | 0x010000; + break; + + case 83: + w5[0] = w5[0] | 0x01000000; + break; + + case 84: + w5[1] = 0x01; + break; + + case 85: + w5[1] = w5[1] | 0x0100; + break; + + case 86: + w5[1] = w5[1] | 0x010000; + break; + + case 87: + w5[1] = w5[1] | 0x01000000; + break; + + case 88: + w5[2] = 0x01; + break; + + case 89: + w5[2] = w5[2] | 0x0100; + break; + + case 90: + w5[2] = w5[2] | 0x010000; + break; + + case 91: + w5[2] = w5[2] | 0x01000000; + break; + + case 92: + w5[3] = 0x01; + break; + + case 93: + w5[3] = w5[3] | 0x0100; + break; + + case 94: + w5[3] = w5[3] | 0x010000; + break; + + case 95: + w5[3] = w5[3] | 0x01000000; + break; + + case 96: + w6[0] = 0x01; + break; + + case 97: + w6[0] = w6[0] | 0x0100; + break; + + case 98: + w6[0] = w6[0] | 0x010000; + break; + + case 99: + w6[0] = w6[0] | 0x01000000; + break; + + case 100: + w6[1] = 0x01; + break; + + case 101: + w6[1] = w6[1] | 0x0100; + break; + + case 102: + w6[1] = w6[1] | 0x010000; + break; + + case 103: + w6[1] = w6[1] | 0x01000000; + break; + + case 104: + w6[2] = 0x01; + break; + + case 105: + w6[2] = w6[2] | 0x0100; + break; + + case 106: + w6[2] = w6[2] | 0x010000; + break; + + case 107: + w6[2] = w6[2] | 0x01000000; + break; + + case 108: + w6[3] = 0x01; + break; + + case 109: + w6[3] = w6[3] | 0x0100; + break; + + case 110: + w6[3] = w6[3] | 0x010000; + break; + + case 111: + w6[3] = w6[3] | 0x01000000; + break; + + case 112: + w7[0] = 0x01; + break; + + case 113: + w7[0] = w7[0] | 0x0100; + break; + + case 114: + w7[0] = w7[0] | 0x010000; + break; + + case 115: + w7[0] = w7[0] | 0x01000000; + break; + + case 116: + w7[1] = 0x01; + break; + + case 117: + w7[1] = w7[1] | 0x0100; + break; + + case 118: + w7[1] = w7[1] | 0x010000; + break; + + case 119: + w7[1] = w7[1] | 0x01000000; + break; + + case 120: + w7[2] = 0x01; + break; + + case 121: + w7[2] = w7[2] | 0x0100; + break; + + case 122: + w7[2] = w7[2] | 0x010000; + break; + + case 123: + w7[2] = w7[2] | 0x01000000; + break; + + case 124: + w7[3] = 0x01; + break; + + case 125: + w7[3] = w7[3] | 0x0100; + break; + + case 126: + w7[3] = w7[3] | 0x010000; + break; + + case 127: + w7[3] = w7[3] | 0x01000000; + break; + } +} + +inline void append_0x02_1x4 (u32x w0[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x02; + break; + + case 1: + w0[0] = w0[0] | 0x0200; + break; + + case 2: + w0[0] = w0[0] | 0x020000; + break; + + case 3: + w0[0] = w0[0] | 0x02000000; + break; + + case 4: + w0[1] = 0x02; + break; + + case 5: + w0[1] = w0[1] | 0x0200; + break; + + case 6: + w0[1] = w0[1] | 0x020000; + break; + + case 7: + w0[1] = w0[1] | 0x02000000; + break; + + case 8: + w0[2] = 0x02; + break; + + case 9: + w0[2] = w0[2] | 0x0200; + break; + + case 10: + w0[2] = w0[2] | 0x020000; + break; + + case 11: + w0[2] = w0[2] | 0x02000000; + break; + + case 12: + w0[3] = 0x02; + break; + + case 13: + w0[3] = w0[3] | 0x0200; + break; + + case 14: + w0[3] = w0[3] | 0x020000; + break; + + case 15: + w0[3] = w0[3] | 0x02000000; + break; + } +} + +inline void append_0x02_2x4 (u32x w0[4], u32x w1[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x02; + break; + + case 1: + w0[0] = w0[0] | 0x0200; + break; + + case 2: + w0[0] = w0[0] | 0x020000; + break; + + case 3: + w0[0] = w0[0] | 0x02000000; + break; + + case 4: + w0[1] = 0x02; + break; + + case 5: + w0[1] = w0[1] | 0x0200; + break; + + case 6: + w0[1] = w0[1] | 0x020000; + break; + + case 7: + w0[1] = w0[1] | 0x02000000; + break; + + case 8: + w0[2] = 0x02; + break; + + case 9: + w0[2] = w0[2] | 0x0200; + break; + + case 10: + w0[2] = w0[2] | 0x020000; + break; + + case 11: + w0[2] = w0[2] | 0x02000000; + break; + + case 12: + w0[3] = 0x02; + break; + + case 13: + w0[3] = w0[3] | 0x0200; + break; + + case 14: + w0[3] = w0[3] | 0x020000; + break; + + case 15: + w0[3] = w0[3] | 0x02000000; + break; + + case 16: + w1[0] = 0x02; + break; + + case 17: + w1[0] = w1[0] | 0x0200; + break; + + case 18: + w1[0] = w1[0] | 0x020000; + break; + + case 19: + w1[0] = w1[0] | 0x02000000; + break; + + case 20: + w1[1] = 0x02; + break; + + case 21: + w1[1] = w1[1] | 0x0200; + break; + + case 22: + w1[1] = w1[1] | 0x020000; + break; + + case 23: + w1[1] = w1[1] | 0x02000000; + break; + + case 24: + w1[2] = 0x02; + break; + + case 25: + w1[2] = w1[2] | 0x0200; + break; + + case 26: + w1[2] = w1[2] | 0x020000; + break; + + case 27: + w1[2] = w1[2] | 0x02000000; + break; + + case 28: + w1[3] = 0x02; + break; + + case 29: + w1[3] = w1[3] | 0x0200; + break; + + case 30: + w1[3] = w1[3] | 0x020000; + break; + + case 31: + w1[3] = w1[3] | 0x02000000; + break; + } +} + +inline void append_0x02_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x02; + break; + + case 1: + w0[0] = w0[0] | 0x0200; + break; + + case 2: + w0[0] = w0[0] | 0x020000; + break; + + case 3: + w0[0] = w0[0] | 0x02000000; + break; + + case 4: + w0[1] = 0x02; + break; + + case 5: + w0[1] = w0[1] | 0x0200; + break; + + case 6: + w0[1] = w0[1] | 0x020000; + break; + + case 7: + w0[1] = w0[1] | 0x02000000; + break; + + case 8: + w0[2] = 0x02; + break; + + case 9: + w0[2] = w0[2] | 0x0200; + break; + + case 10: + w0[2] = w0[2] | 0x020000; + break; + + case 11: + w0[2] = w0[2] | 0x02000000; + break; + + case 12: + w0[3] = 0x02; + break; + + case 13: + w0[3] = w0[3] | 0x0200; + break; + + case 14: + w0[3] = w0[3] | 0x020000; + break; + + case 15: + w0[3] = w0[3] | 0x02000000; + break; + + case 16: + w1[0] = 0x02; + break; + + case 17: + w1[0] = w1[0] | 0x0200; + break; + + case 18: + w1[0] = w1[0] | 0x020000; + break; + + case 19: + w1[0] = w1[0] | 0x02000000; + break; + + case 20: + w1[1] = 0x02; + break; + + case 21: + w1[1] = w1[1] | 0x0200; + break; + + case 22: + w1[1] = w1[1] | 0x020000; + break; + + case 23: + w1[1] = w1[1] | 0x02000000; + break; + + case 24: + w1[2] = 0x02; + break; + + case 25: + w1[2] = w1[2] | 0x0200; + break; + + case 26: + w1[2] = w1[2] | 0x020000; + break; + + case 27: + w1[2] = w1[2] | 0x02000000; + break; + + case 28: + w1[3] = 0x02; + break; + + case 29: + w1[3] = w1[3] | 0x0200; + break; + + case 30: + w1[3] = w1[3] | 0x020000; + break; + + case 31: + w1[3] = w1[3] | 0x02000000; + break; + + case 32: + w2[0] = 0x02; + break; + + case 33: + w2[0] = w2[0] | 0x0200; + break; + + case 34: + w2[0] = w2[0] | 0x020000; + break; + + case 35: + w2[0] = w2[0] | 0x02000000; + break; + + case 36: + w2[1] = 0x02; + break; + + case 37: + w2[1] = w2[1] | 0x0200; + break; + + case 38: + w2[1] = w2[1] | 0x020000; + break; + + case 39: + w2[1] = w2[1] | 0x02000000; + break; + + case 40: + w2[2] = 0x02; + break; + + case 41: + w2[2] = w2[2] | 0x0200; + break; + + case 42: + w2[2] = w2[2] | 0x020000; + break; + + case 43: + w2[2] = w2[2] | 0x02000000; + break; + + case 44: + w2[3] = 0x02; + break; + + case 45: + w2[3] = w2[3] | 0x0200; + break; + + case 46: + w2[3] = w2[3] | 0x020000; + break; + + case 47: + w2[3] = w2[3] | 0x02000000; + break; + } +} + +inline void append_0x02_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x02; + break; + + case 1: + w0[0] = w0[0] | 0x0200; + break; + + case 2: + w0[0] = w0[0] | 0x020000; + break; + + case 3: + w0[0] = w0[0] | 0x02000000; + break; + + case 4: + w0[1] = 0x02; + break; + + case 5: + w0[1] = w0[1] | 0x0200; + break; + + case 6: + w0[1] = w0[1] | 0x020000; + break; + + case 7: + w0[1] = w0[1] | 0x02000000; + break; + + case 8: + w0[2] = 0x02; + break; + + case 9: + w0[2] = w0[2] | 0x0200; + break; + + case 10: + w0[2] = w0[2] | 0x020000; + break; + + case 11: + w0[2] = w0[2] | 0x02000000; + break; + + case 12: + w0[3] = 0x02; + break; + + case 13: + w0[3] = w0[3] | 0x0200; + break; + + case 14: + w0[3] = w0[3] | 0x020000; + break; + + case 15: + w0[3] = w0[3] | 0x02000000; + break; + + case 16: + w1[0] = 0x02; + break; + + case 17: + w1[0] = w1[0] | 0x0200; + break; + + case 18: + w1[0] = w1[0] | 0x020000; + break; + + case 19: + w1[0] = w1[0] | 0x02000000; + break; + + case 20: + w1[1] = 0x02; + break; + + case 21: + w1[1] = w1[1] | 0x0200; + break; + + case 22: + w1[1] = w1[1] | 0x020000; + break; + + case 23: + w1[1] = w1[1] | 0x02000000; + break; + + case 24: + w1[2] = 0x02; + break; + + case 25: + w1[2] = w1[2] | 0x0200; + break; + + case 26: + w1[2] = w1[2] | 0x020000; + break; + + case 27: + w1[2] = w1[2] | 0x02000000; + break; + + case 28: + w1[3] = 0x02; + break; + + case 29: + w1[3] = w1[3] | 0x0200; + break; + + case 30: + w1[3] = w1[3] | 0x020000; + break; + + case 31: + w1[3] = w1[3] | 0x02000000; + break; + + case 32: + w2[0] = 0x02; + break; + + case 33: + w2[0] = w2[0] | 0x0200; + break; + + case 34: + w2[0] = w2[0] | 0x020000; + break; + + case 35: + w2[0] = w2[0] | 0x02000000; + break; + + case 36: + w2[1] = 0x02; + break; + + case 37: + w2[1] = w2[1] | 0x0200; + break; + + case 38: + w2[1] = w2[1] | 0x020000; + break; + + case 39: + w2[1] = w2[1] | 0x02000000; + break; + + case 40: + w2[2] = 0x02; + break; + + case 41: + w2[2] = w2[2] | 0x0200; + break; + + case 42: + w2[2] = w2[2] | 0x020000; + break; + + case 43: + w2[2] = w2[2] | 0x02000000; + break; + + case 44: + w2[3] = 0x02; + break; + + case 45: + w2[3] = w2[3] | 0x0200; + break; + + case 46: + w2[3] = w2[3] | 0x020000; + break; + + case 47: + w2[3] = w2[3] | 0x02000000; + break; + + case 48: + w3[0] = 0x02; + break; + + case 49: + w3[0] = w3[0] | 0x0200; + break; + + case 50: + w3[0] = w3[0] | 0x020000; + break; + + case 51: + w3[0] = w3[0] | 0x02000000; + break; + + case 52: + w3[1] = 0x02; + break; + + case 53: + w3[1] = w3[1] | 0x0200; + break; + + case 54: + w3[1] = w3[1] | 0x020000; + break; + + case 55: + w3[1] = w3[1] | 0x02000000; + break; + + case 56: + w3[2] = 0x02; + break; + + case 57: + w3[2] = w3[2] | 0x0200; + break; + + case 58: + w3[2] = w3[2] | 0x020000; + break; + + case 59: + w3[2] = w3[2] | 0x02000000; + break; + + case 60: + w3[3] = 0x02; + break; + + case 61: + w3[3] = w3[3] | 0x0200; + break; + + case 62: + w3[3] = w3[3] | 0x020000; + break; + + case 63: + w3[3] = w3[3] | 0x02000000; + break; + } +} + +inline void append_0x02_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x02; + break; + + case 1: + w0[0] = w0[0] | 0x0200; + break; + + case 2: + w0[0] = w0[0] | 0x020000; + break; + + case 3: + w0[0] = w0[0] | 0x02000000; + break; + + case 4: + w0[1] = 0x02; + break; + + case 5: + w0[1] = w0[1] | 0x0200; + break; + + case 6: + w0[1] = w0[1] | 0x020000; + break; + + case 7: + w0[1] = w0[1] | 0x02000000; + break; + + case 8: + w0[2] = 0x02; + break; + + case 9: + w0[2] = w0[2] | 0x0200; + break; + + case 10: + w0[2] = w0[2] | 0x020000; + break; + + case 11: + w0[2] = w0[2] | 0x02000000; + break; + + case 12: + w0[3] = 0x02; + break; + + case 13: + w0[3] = w0[3] | 0x0200; + break; + + case 14: + w0[3] = w0[3] | 0x020000; + break; + + case 15: + w0[3] = w0[3] | 0x02000000; + break; + + case 16: + w1[0] = 0x02; + break; + + case 17: + w1[0] = w1[0] | 0x0200; + break; + + case 18: + w1[0] = w1[0] | 0x020000; + break; + + case 19: + w1[0] = w1[0] | 0x02000000; + break; + + case 20: + w1[1] = 0x02; + break; + + case 21: + w1[1] = w1[1] | 0x0200; + break; + + case 22: + w1[1] = w1[1] | 0x020000; + break; + + case 23: + w1[1] = w1[1] | 0x02000000; + break; + + case 24: + w1[2] = 0x02; + break; + + case 25: + w1[2] = w1[2] | 0x0200; + break; + + case 26: + w1[2] = w1[2] | 0x020000; + break; + + case 27: + w1[2] = w1[2] | 0x02000000; + break; + + case 28: + w1[3] = 0x02; + break; + + case 29: + w1[3] = w1[3] | 0x0200; + break; + + case 30: + w1[3] = w1[3] | 0x020000; + break; + + case 31: + w1[3] = w1[3] | 0x02000000; + break; + + case 32: + w2[0] = 0x02; + break; + + case 33: + w2[0] = w2[0] | 0x0200; + break; + + case 34: + w2[0] = w2[0] | 0x020000; + break; + + case 35: + w2[0] = w2[0] | 0x02000000; + break; + + case 36: + w2[1] = 0x02; + break; + + case 37: + w2[1] = w2[1] | 0x0200; + break; + + case 38: + w2[1] = w2[1] | 0x020000; + break; + + case 39: + w2[1] = w2[1] | 0x02000000; + break; + + case 40: + w2[2] = 0x02; + break; + + case 41: + w2[2] = w2[2] | 0x0200; + break; + + case 42: + w2[2] = w2[2] | 0x020000; + break; + + case 43: + w2[2] = w2[2] | 0x02000000; + break; + + case 44: + w2[3] = 0x02; + break; + + case 45: + w2[3] = w2[3] | 0x0200; + break; + + case 46: + w2[3] = w2[3] | 0x020000; + break; + + case 47: + w2[3] = w2[3] | 0x02000000; + break; + + case 48: + w3[0] = 0x02; + break; + + case 49: + w3[0] = w3[0] | 0x0200; + break; + + case 50: + w3[0] = w3[0] | 0x020000; + break; + + case 51: + w3[0] = w3[0] | 0x02000000; + break; + + case 52: + w3[1] = 0x02; + break; + + case 53: + w3[1] = w3[1] | 0x0200; + break; + + case 54: + w3[1] = w3[1] | 0x020000; + break; + + case 55: + w3[1] = w3[1] | 0x02000000; + break; + + case 56: + w3[2] = 0x02; + break; + + case 57: + w3[2] = w3[2] | 0x0200; + break; + + case 58: + w3[2] = w3[2] | 0x020000; + break; + + case 59: + w3[2] = w3[2] | 0x02000000; + break; + + case 60: + w3[3] = 0x02; + break; + + case 61: + w3[3] = w3[3] | 0x0200; + break; + + case 62: + w3[3] = w3[3] | 0x020000; + break; + + case 63: + w3[3] = w3[3] | 0x02000000; + break; + + case 64: + w4[0] = 0x02; + break; + + case 65: + w4[0] = w4[0] | 0x0200; + break; + + case 66: + w4[0] = w4[0] | 0x020000; + break; + + case 67: + w4[0] = w4[0] | 0x02000000; + break; + + case 68: + w4[1] = 0x02; + break; + + case 69: + w4[1] = w4[1] | 0x0200; + break; + + case 70: + w4[1] = w4[1] | 0x020000; + break; + + case 71: + w4[1] = w4[1] | 0x02000000; + break; + + case 72: + w4[2] = 0x02; + break; + + case 73: + w4[2] = w4[2] | 0x0200; + break; + + case 74: + w4[2] = w4[2] | 0x020000; + break; + + case 75: + w4[2] = w4[2] | 0x02000000; + break; + + case 76: + w4[3] = 0x02; + break; + + case 77: + w4[3] = w4[3] | 0x0200; + break; + + case 78: + w4[3] = w4[3] | 0x020000; + break; + + case 79: + w4[3] = w4[3] | 0x02000000; + break; + + case 80: + w5[0] = 0x02; + break; + + case 81: + w5[0] = w5[0] | 0x0200; + break; + + case 82: + w5[0] = w5[0] | 0x020000; + break; + + case 83: + w5[0] = w5[0] | 0x02000000; + break; + + case 84: + w5[1] = 0x02; + break; + + case 85: + w5[1] = w5[1] | 0x0200; + break; + + case 86: + w5[1] = w5[1] | 0x020000; + break; + + case 87: + w5[1] = w5[1] | 0x02000000; + break; + + case 88: + w5[2] = 0x02; + break; + + case 89: + w5[2] = w5[2] | 0x0200; + break; + + case 90: + w5[2] = w5[2] | 0x020000; + break; + + case 91: + w5[2] = w5[2] | 0x02000000; + break; + + case 92: + w5[3] = 0x02; + break; + + case 93: + w5[3] = w5[3] | 0x0200; + break; + + case 94: + w5[3] = w5[3] | 0x020000; + break; + + case 95: + w5[3] = w5[3] | 0x02000000; + break; + + case 96: + w6[0] = 0x02; + break; + + case 97: + w6[0] = w6[0] | 0x0200; + break; + + case 98: + w6[0] = w6[0] | 0x020000; + break; + + case 99: + w6[0] = w6[0] | 0x02000000; + break; + + case 100: + w6[1] = 0x02; + break; + + case 101: + w6[1] = w6[1] | 0x0200; + break; + + case 102: + w6[1] = w6[1] | 0x020000; + break; + + case 103: + w6[1] = w6[1] | 0x02000000; + break; + + case 104: + w6[2] = 0x02; + break; + + case 105: + w6[2] = w6[2] | 0x0200; + break; + + case 106: + w6[2] = w6[2] | 0x020000; + break; + + case 107: + w6[2] = w6[2] | 0x02000000; + break; + + case 108: + w6[3] = 0x02; + break; + + case 109: + w6[3] = w6[3] | 0x0200; + break; + + case 110: + w6[3] = w6[3] | 0x020000; + break; + + case 111: + w6[3] = w6[3] | 0x02000000; + break; + + case 112: + w7[0] = 0x02; + break; + + case 113: + w7[0] = w7[0] | 0x0200; + break; + + case 114: + w7[0] = w7[0] | 0x020000; + break; + + case 115: + w7[0] = w7[0] | 0x02000000; + break; + + case 116: + w7[1] = 0x02; + break; + + case 117: + w7[1] = w7[1] | 0x0200; + break; + + case 118: + w7[1] = w7[1] | 0x020000; + break; + + case 119: + w7[1] = w7[1] | 0x02000000; + break; + + case 120: + w7[2] = 0x02; + break; + + case 121: + w7[2] = w7[2] | 0x0200; + break; + + case 122: + w7[2] = w7[2] | 0x020000; + break; + + case 123: + w7[2] = w7[2] | 0x02000000; + break; + + case 124: + w7[3] = 0x02; + break; + + case 125: + w7[3] = w7[3] | 0x0200; + break; + + case 126: + w7[3] = w7[3] | 0x020000; + break; + + case 127: + w7[3] = w7[3] | 0x02000000; + break; + } +} + +inline void append_0x80_1x4 (u32x w0[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x80; + break; + + case 1: + w0[0] = w0[0] | 0x8000; + break; + + case 2: + w0[0] = w0[0] | 0x800000; + break; + + case 3: + w0[0] = w0[0] | 0x80000000; + break; + + case 4: + w0[1] = 0x80; + break; + + case 5: + w0[1] = w0[1] | 0x8000; + break; + + case 6: + w0[1] = w0[1] | 0x800000; + break; + + case 7: + w0[1] = w0[1] | 0x80000000; + break; + + case 8: + w0[2] = 0x80; + break; + + case 9: + w0[2] = w0[2] | 0x8000; + break; + + case 10: + w0[2] = w0[2] | 0x800000; + break; + + case 11: + w0[2] = w0[2] | 0x80000000; + break; + + case 12: + w0[3] = 0x80; + break; + + case 13: + w0[3] = w0[3] | 0x8000; + break; + + case 14: + w0[3] = w0[3] | 0x800000; + break; + + case 15: + w0[3] = w0[3] | 0x80000000; + break; + } +} + +inline void append_0x80_2x4 (u32x w0[4], u32x w1[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x80; + break; + + case 1: + w0[0] = w0[0] | 0x8000; + break; + + case 2: + w0[0] = w0[0] | 0x800000; + break; + + case 3: + w0[0] = w0[0] | 0x80000000; + break; + + case 4: + w0[1] = 0x80; + break; + + case 5: + w0[1] = w0[1] | 0x8000; + break; + + case 6: + w0[1] = w0[1] | 0x800000; + break; + + case 7: + w0[1] = w0[1] | 0x80000000; + break; + + case 8: + w0[2] = 0x80; + break; + + case 9: + w0[2] = w0[2] | 0x8000; + break; + + case 10: + w0[2] = w0[2] | 0x800000; + break; + + case 11: + w0[2] = w0[2] | 0x80000000; + break; + + case 12: + w0[3] = 0x80; + break; + + case 13: + w0[3] = w0[3] | 0x8000; + break; + + case 14: + w0[3] = w0[3] | 0x800000; + break; + + case 15: + w0[3] = w0[3] | 0x80000000; + break; + + case 16: + w1[0] = 0x80; + break; + + case 17: + w1[0] = w1[0] | 0x8000; + break; + + case 18: + w1[0] = w1[0] | 0x800000; + break; + + case 19: + w1[0] = w1[0] | 0x80000000; + break; + + case 20: + w1[1] = 0x80; + break; + + case 21: + w1[1] = w1[1] | 0x8000; + break; + + case 22: + w1[1] = w1[1] | 0x800000; + break; + + case 23: + w1[1] = w1[1] | 0x80000000; + break; + + case 24: + w1[2] = 0x80; + break; + + case 25: + w1[2] = w1[2] | 0x8000; + break; + + case 26: + w1[2] = w1[2] | 0x800000; + break; + + case 27: + w1[2] = w1[2] | 0x80000000; + break; + + case 28: + w1[3] = 0x80; + break; + + case 29: + w1[3] = w1[3] | 0x8000; + break; + + case 30: + w1[3] = w1[3] | 0x800000; + break; + + case 31: + w1[3] = w1[3] | 0x80000000; + break; + } +} + +inline void append_0x80_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x80; + break; + + case 1: + w0[0] = w0[0] | 0x8000; + break; + + case 2: + w0[0] = w0[0] | 0x800000; + break; + + case 3: + w0[0] = w0[0] | 0x80000000; + break; + + case 4: + w0[1] = 0x80; + break; + + case 5: + w0[1] = w0[1] | 0x8000; + break; + + case 6: + w0[1] = w0[1] | 0x800000; + break; + + case 7: + w0[1] = w0[1] | 0x80000000; + break; + + case 8: + w0[2] = 0x80; + break; + + case 9: + w0[2] = w0[2] | 0x8000; + break; + + case 10: + w0[2] = w0[2] | 0x800000; + break; + + case 11: + w0[2] = w0[2] | 0x80000000; + break; + + case 12: + w0[3] = 0x80; + break; + + case 13: + w0[3] = w0[3] | 0x8000; + break; + + case 14: + w0[3] = w0[3] | 0x800000; + break; + + case 15: + w0[3] = w0[3] | 0x80000000; + break; + + case 16: + w1[0] = 0x80; + break; + + case 17: + w1[0] = w1[0] | 0x8000; + break; + + case 18: + w1[0] = w1[0] | 0x800000; + break; + + case 19: + w1[0] = w1[0] | 0x80000000; + break; + + case 20: + w1[1] = 0x80; + break; + + case 21: + w1[1] = w1[1] | 0x8000; + break; + + case 22: + w1[1] = w1[1] | 0x800000; + break; + + case 23: + w1[1] = w1[1] | 0x80000000; + break; + + case 24: + w1[2] = 0x80; + break; + + case 25: + w1[2] = w1[2] | 0x8000; + break; + + case 26: + w1[2] = w1[2] | 0x800000; + break; + + case 27: + w1[2] = w1[2] | 0x80000000; + break; + + case 28: + w1[3] = 0x80; + break; + + case 29: + w1[3] = w1[3] | 0x8000; + break; + + case 30: + w1[3] = w1[3] | 0x800000; + break; + + case 31: + w1[3] = w1[3] | 0x80000000; + break; + + case 32: + w2[0] = 0x80; + break; + + case 33: + w2[0] = w2[0] | 0x8000; + break; + + case 34: + w2[0] = w2[0] | 0x800000; + break; + + case 35: + w2[0] = w2[0] | 0x80000000; + break; + + case 36: + w2[1] = 0x80; + break; + + case 37: + w2[1] = w2[1] | 0x8000; + break; + + case 38: + w2[1] = w2[1] | 0x800000; + break; + + case 39: + w2[1] = w2[1] | 0x80000000; + break; + + case 40: + w2[2] = 0x80; + break; + + case 41: + w2[2] = w2[2] | 0x8000; + break; + + case 42: + w2[2] = w2[2] | 0x800000; + break; + + case 43: + w2[2] = w2[2] | 0x80000000; + break; + + case 44: + w2[3] = 0x80; + break; + + case 45: + w2[3] = w2[3] | 0x8000; + break; + + case 46: + w2[3] = w2[3] | 0x800000; + break; + + case 47: + w2[3] = w2[3] | 0x80000000; + break; + } +} + +inline void append_0x80_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x80; + break; + + case 1: + w0[0] = w0[0] | 0x8000; + break; + + case 2: + w0[0] = w0[0] | 0x800000; + break; + + case 3: + w0[0] = w0[0] | 0x80000000; + break; + + case 4: + w0[1] = 0x80; + break; + + case 5: + w0[1] = w0[1] | 0x8000; + break; + + case 6: + w0[1] = w0[1] | 0x800000; + break; + + case 7: + w0[1] = w0[1] | 0x80000000; + break; + + case 8: + w0[2] = 0x80; + break; + + case 9: + w0[2] = w0[2] | 0x8000; + break; + + case 10: + w0[2] = w0[2] | 0x800000; + break; + + case 11: + w0[2] = w0[2] | 0x80000000; + break; + + case 12: + w0[3] = 0x80; + break; + + case 13: + w0[3] = w0[3] | 0x8000; + break; + + case 14: + w0[3] = w0[3] | 0x800000; + break; + + case 15: + w0[3] = w0[3] | 0x80000000; + break; + + case 16: + w1[0] = 0x80; + break; + + case 17: + w1[0] = w1[0] | 0x8000; + break; + + case 18: + w1[0] = w1[0] | 0x800000; + break; + + case 19: + w1[0] = w1[0] | 0x80000000; + break; + + case 20: + w1[1] = 0x80; + break; + + case 21: + w1[1] = w1[1] | 0x8000; + break; + + case 22: + w1[1] = w1[1] | 0x800000; + break; + + case 23: + w1[1] = w1[1] | 0x80000000; + break; + + case 24: + w1[2] = 0x80; + break; + + case 25: + w1[2] = w1[2] | 0x8000; + break; + + case 26: + w1[2] = w1[2] | 0x800000; + break; + + case 27: + w1[2] = w1[2] | 0x80000000; + break; + + case 28: + w1[3] = 0x80; + break; + + case 29: + w1[3] = w1[3] | 0x8000; + break; + + case 30: + w1[3] = w1[3] | 0x800000; + break; + + case 31: + w1[3] = w1[3] | 0x80000000; + break; + + case 32: + w2[0] = 0x80; + break; + + case 33: + w2[0] = w2[0] | 0x8000; + break; + + case 34: + w2[0] = w2[0] | 0x800000; + break; + + case 35: + w2[0] = w2[0] | 0x80000000; + break; + + case 36: + w2[1] = 0x80; + break; + + case 37: + w2[1] = w2[1] | 0x8000; + break; + + case 38: + w2[1] = w2[1] | 0x800000; + break; + + case 39: + w2[1] = w2[1] | 0x80000000; + break; + + case 40: + w2[2] = 0x80; + break; + + case 41: + w2[2] = w2[2] | 0x8000; + break; + + case 42: + w2[2] = w2[2] | 0x800000; + break; + + case 43: + w2[2] = w2[2] | 0x80000000; + break; + + case 44: + w2[3] = 0x80; + break; + + case 45: + w2[3] = w2[3] | 0x8000; + break; + + case 46: + w2[3] = w2[3] | 0x800000; + break; + + case 47: + w2[3] = w2[3] | 0x80000000; + break; + + case 48: + w3[0] = 0x80; + break; + + case 49: + w3[0] = w3[0] | 0x8000; + break; + + case 50: + w3[0] = w3[0] | 0x800000; + break; + + case 51: + w3[0] = w3[0] | 0x80000000; + break; + + case 52: + w3[1] = 0x80; + break; + + case 53: + w3[1] = w3[1] | 0x8000; + break; + + case 54: + w3[1] = w3[1] | 0x800000; + break; + + case 55: + w3[1] = w3[1] | 0x80000000; + break; + + case 56: + w3[2] = 0x80; + break; + + case 57: + w3[2] = w3[2] | 0x8000; + break; + + case 58: + w3[2] = w3[2] | 0x800000; + break; + + case 59: + w3[2] = w3[2] | 0x80000000; + break; + + case 60: + w3[3] = 0x80; + break; + + case 61: + w3[3] = w3[3] | 0x8000; + break; + + case 62: + w3[3] = w3[3] | 0x800000; + break; + + case 63: + w3[3] = w3[3] | 0x80000000; + break; + } +} + +inline void append_0x80_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x80; + break; + + case 1: + w0[0] = w0[0] | 0x8000; + break; + + case 2: + w0[0] = w0[0] | 0x800000; + break; + + case 3: + w0[0] = w0[0] | 0x80000000; + break; + + case 4: + w0[1] = 0x80; + break; + + case 5: + w0[1] = w0[1] | 0x8000; + break; + + case 6: + w0[1] = w0[1] | 0x800000; + break; + + case 7: + w0[1] = w0[1] | 0x80000000; + break; + + case 8: + w0[2] = 0x80; + break; + + case 9: + w0[2] = w0[2] | 0x8000; + break; + + case 10: + w0[2] = w0[2] | 0x800000; + break; + + case 11: + w0[2] = w0[2] | 0x80000000; + break; + + case 12: + w0[3] = 0x80; + break; + + case 13: + w0[3] = w0[3] | 0x8000; + break; + + case 14: + w0[3] = w0[3] | 0x800000; + break; + + case 15: + w0[3] = w0[3] | 0x80000000; + break; + + case 16: + w1[0] = 0x80; + break; + + case 17: + w1[0] = w1[0] | 0x8000; + break; + + case 18: + w1[0] = w1[0] | 0x800000; + break; + + case 19: + w1[0] = w1[0] | 0x80000000; + break; + + case 20: + w1[1] = 0x80; + break; + + case 21: + w1[1] = w1[1] | 0x8000; + break; + + case 22: + w1[1] = w1[1] | 0x800000; + break; + + case 23: + w1[1] = w1[1] | 0x80000000; + break; + + case 24: + w1[2] = 0x80; + break; + + case 25: + w1[2] = w1[2] | 0x8000; + break; + + case 26: + w1[2] = w1[2] | 0x800000; + break; + + case 27: + w1[2] = w1[2] | 0x80000000; + break; + + case 28: + w1[3] = 0x80; + break; + + case 29: + w1[3] = w1[3] | 0x8000; + break; + + case 30: + w1[3] = w1[3] | 0x800000; + break; + + case 31: + w1[3] = w1[3] | 0x80000000; + break; + + case 32: + w2[0] = 0x80; + break; + + case 33: + w2[0] = w2[0] | 0x8000; + break; + + case 34: + w2[0] = w2[0] | 0x800000; + break; + + case 35: + w2[0] = w2[0] | 0x80000000; + break; + + case 36: + w2[1] = 0x80; + break; + + case 37: + w2[1] = w2[1] | 0x8000; + break; + + case 38: + w2[1] = w2[1] | 0x800000; + break; + + case 39: + w2[1] = w2[1] | 0x80000000; + break; + + case 40: + w2[2] = 0x80; + break; + + case 41: + w2[2] = w2[2] | 0x8000; + break; + + case 42: + w2[2] = w2[2] | 0x800000; + break; + + case 43: + w2[2] = w2[2] | 0x80000000; + break; + + case 44: + w2[3] = 0x80; + break; + + case 45: + w2[3] = w2[3] | 0x8000; + break; + + case 46: + w2[3] = w2[3] | 0x800000; + break; + + case 47: + w2[3] = w2[3] | 0x80000000; + break; + + case 48: + w3[0] = 0x80; + break; + + case 49: + w3[0] = w3[0] | 0x8000; + break; + + case 50: + w3[0] = w3[0] | 0x800000; + break; + + case 51: + w3[0] = w3[0] | 0x80000000; + break; + + case 52: + w3[1] = 0x80; + break; + + case 53: + w3[1] = w3[1] | 0x8000; + break; + + case 54: + w3[1] = w3[1] | 0x800000; + break; + + case 55: + w3[1] = w3[1] | 0x80000000; + break; + + case 56: + w3[2] = 0x80; + break; + + case 57: + w3[2] = w3[2] | 0x8000; + break; + + case 58: + w3[2] = w3[2] | 0x800000; + break; + + case 59: + w3[2] = w3[2] | 0x80000000; + break; + + case 60: + w3[3] = 0x80; + break; + + case 61: + w3[3] = w3[3] | 0x8000; + break; + + case 62: + w3[3] = w3[3] | 0x800000; + break; + + case 63: + w3[3] = w3[3] | 0x80000000; + break; + + case 64: + w4[0] = 0x80; + break; + + case 65: + w4[0] = w4[0] | 0x8000; + break; + + case 66: + w4[0] = w4[0] | 0x800000; + break; + + case 67: + w4[0] = w4[0] | 0x80000000; + break; + + case 68: + w4[1] = 0x80; + break; + + case 69: + w4[1] = w4[1] | 0x8000; + break; + + case 70: + w4[1] = w4[1] | 0x800000; + break; + + case 71: + w4[1] = w4[1] | 0x80000000; + break; + + case 72: + w4[2] = 0x80; + break; + + case 73: + w4[2] = w4[2] | 0x8000; + break; + + case 74: + w4[2] = w4[2] | 0x800000; + break; + + case 75: + w4[2] = w4[2] | 0x80000000; + break; + + case 76: + w4[3] = 0x80; + break; + + case 77: + w4[3] = w4[3] | 0x8000; + break; + + case 78: + w4[3] = w4[3] | 0x800000; + break; + + case 79: + w4[3] = w4[3] | 0x80000000; + break; + + case 80: + w5[0] = 0x80; + break; + + case 81: + w5[0] = w5[0] | 0x8000; + break; + + case 82: + w5[0] = w5[0] | 0x800000; + break; + + case 83: + w5[0] = w5[0] | 0x80000000; + break; + + case 84: + w5[1] = 0x80; + break; + + case 85: + w5[1] = w5[1] | 0x8000; + break; + + case 86: + w5[1] = w5[1] | 0x800000; + break; + + case 87: + w5[1] = w5[1] | 0x80000000; + break; + + case 88: + w5[2] = 0x80; + break; + + case 89: + w5[2] = w5[2] | 0x8000; + break; + + case 90: + w5[2] = w5[2] | 0x800000; + break; + + case 91: + w5[2] = w5[2] | 0x80000000; + break; + + case 92: + w5[3] = 0x80; + break; + + case 93: + w5[3] = w5[3] | 0x8000; + break; + + case 94: + w5[3] = w5[3] | 0x800000; + break; + + case 95: + w5[3] = w5[3] | 0x80000000; + break; + + case 96: + w6[0] = 0x80; + break; + + case 97: + w6[0] = w6[0] | 0x8000; + break; + + case 98: + w6[0] = w6[0] | 0x800000; + break; + + case 99: + w6[0] = w6[0] | 0x80000000; + break; + + case 100: + w6[1] = 0x80; + break; + + case 101: + w6[1] = w6[1] | 0x8000; + break; + + case 102: + w6[1] = w6[1] | 0x800000; + break; + + case 103: + w6[1] = w6[1] | 0x80000000; + break; + + case 104: + w6[2] = 0x80; + break; + + case 105: + w6[2] = w6[2] | 0x8000; + break; + + case 106: + w6[2] = w6[2] | 0x800000; + break; + + case 107: + w6[2] = w6[2] | 0x80000000; + break; + + case 108: + w6[3] = 0x80; + break; + + case 109: + w6[3] = w6[3] | 0x8000; + break; + + case 110: + w6[3] = w6[3] | 0x800000; + break; + + case 111: + w6[3] = w6[3] | 0x80000000; + break; + + case 112: + w7[0] = 0x80; + break; + + case 113: + w7[0] = w7[0] | 0x8000; + break; + + case 114: + w7[0] = w7[0] | 0x800000; + break; + + case 115: + w7[0] = w7[0] | 0x80000000; + break; + + case 116: + w7[1] = 0x80; + break; + + case 117: + w7[1] = w7[1] | 0x8000; + break; + + case 118: + w7[1] = w7[1] | 0x800000; + break; + + case 119: + w7[1] = w7[1] | 0x80000000; + break; + + case 120: + w7[2] = 0x80; + break; + + case 121: + w7[2] = w7[2] | 0x8000; + break; + + case 122: + w7[2] = w7[2] | 0x800000; + break; + + case 123: + w7[2] = w7[2] | 0x80000000; + break; + + case 124: + w7[3] = 0x80; + break; + + case 125: + w7[3] = w7[3] | 0x8000; + break; + + case 126: + w7[3] = w7[3] | 0x800000; + break; + + case 127: + w7[3] = w7[3] | 0x80000000; + break; + } +} + +inline void append_0x80_1x16 (u32x w[16], const u32 offset) +{ + switch (offset) + { + case 0: + w[ 0] = 0x80; + break; + + case 1: + w[ 0] = w[ 0] | 0x8000; + break; + + case 2: + w[ 0] = w[ 0] | 0x800000; + break; + + case 3: + w[ 0] = w[ 0] | 0x80000000; + break; + + case 4: + w[ 1] = 0x80; + break; + + case 5: + w[ 1] = w[ 1] | 0x8000; + break; + + case 6: + w[ 1] = w[ 1] | 0x800000; + break; + + case 7: + w[ 1] = w[ 1] | 0x80000000; + break; + + case 8: + w[ 2] = 0x80; + break; + + case 9: + w[ 2] = w[ 2] | 0x8000; + break; + + case 10: + w[ 2] = w[ 2] | 0x800000; + break; + + case 11: + w[ 2] = w[ 2] | 0x80000000; + break; + + case 12: + w[ 3] = 0x80; + break; + + case 13: + w[ 3] = w[ 3] | 0x8000; + break; + + case 14: + w[ 3] = w[ 3] | 0x800000; + break; + + case 15: + w[ 3] = w[ 3] | 0x80000000; + break; + + case 16: + w[ 4] = 0x80; + break; + + case 17: + w[ 4] = w[ 4] | 0x8000; + break; + + case 18: + w[ 4] = w[ 4] | 0x800000; + break; + + case 19: + w[ 4] = w[ 4] | 0x80000000; + break; + + case 20: + w[ 5] = 0x80; + break; + + case 21: + w[ 5] = w[ 5] | 0x8000; + break; + + case 22: + w[ 5] = w[ 5] | 0x800000; + break; + + case 23: + w[ 5] = w[ 5] | 0x80000000; + break; + + case 24: + w[ 6] = 0x80; + break; + + case 25: + w[ 6] = w[ 6] | 0x8000; + break; + + case 26: + w[ 6] = w[ 6] | 0x800000; + break; + + case 27: + w[ 6] = w[ 6] | 0x80000000; + break; + + case 28: + w[ 7] = 0x80; + break; + + case 29: + w[ 7] = w[ 7] | 0x8000; + break; + + case 30: + w[ 7] = w[ 7] | 0x800000; + break; + + case 31: + w[ 7] = w[ 7] | 0x80000000; + break; + + case 32: + w[ 8] = 0x80; + break; + + case 33: + w[ 8] = w[ 8] | 0x8000; + break; + + case 34: + w[ 8] = w[ 8] | 0x800000; + break; + + case 35: + w[ 8] = w[ 8] | 0x80000000; + break; + + case 36: + w[ 9] = 0x80; + break; + + case 37: + w[ 9] = w[ 9] | 0x8000; + break; + + case 38: + w[ 9] = w[ 9] | 0x800000; + break; + + case 39: + w[ 9] = w[ 9] | 0x80000000; + break; + + case 40: + w[10] = 0x80; + break; + + case 41: + w[10] = w[10] | 0x8000; + break; + + case 42: + w[10] = w[10] | 0x800000; + break; + + case 43: + w[10] = w[10] | 0x80000000; + break; + + case 44: + w[11] = 0x80; + break; + + case 45: + w[11] = w[11] | 0x8000; + break; + + case 46: + w[11] = w[11] | 0x800000; + break; + + case 47: + w[11] = w[11] | 0x80000000; + break; + + case 48: + w[12] = 0x80; + break; + + case 49: + w[12] = w[12] | 0x8000; + break; + + case 50: + w[12] = w[12] | 0x800000; + break; + + case 51: + w[12] = w[12] | 0x80000000; + break; + + case 52: + w[13] = 0x80; + break; + + case 53: + w[13] = w[13] | 0x8000; + break; + + case 54: + w[13] = w[13] | 0x800000; + break; + + case 55: + w[13] = w[13] | 0x80000000; + break; + + case 56: + w[14] = 0x80; + break; + + case 57: + w[14] = w[14] | 0x8000; + break; + + case 58: + w[14] = w[14] | 0x800000; + break; + + case 59: + w[14] = w[14] | 0x80000000; + break; + + case 60: + w[15] = 0x80; + break; + + case 61: + w[15] = w[15] | 0x8000; + break; + + case 62: + w[15] = w[15] | 0x800000; + break; + + case 63: + w[15] = w[15] | 0x80000000; + break; + } +} + +inline void switch_buffer_by_offset_le (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) +{ + #if defined IS_AMD || defined IS_GENERIC + const int offset_mod_4 = offset & 3; + + const int offset_minus_4 = 4 - offset; + + switch (offset / 4) + { + case 0: + w3[2] = amd_bytealign ( 0, w3[1], offset_minus_4); + w3[1] = amd_bytealign (w3[1], w3[0], offset_minus_4); + w3[0] = amd_bytealign (w3[0], w2[3], offset_minus_4); + w2[3] = amd_bytealign (w2[3], w2[2], offset_minus_4); + w2[2] = amd_bytealign (w2[2], w2[1], offset_minus_4); + w2[1] = amd_bytealign (w2[1], w2[0], offset_minus_4); + w2[0] = amd_bytealign (w2[0], w1[3], offset_minus_4); + w1[3] = amd_bytealign (w1[3], w1[2], offset_minus_4); + w1[2] = amd_bytealign (w1[2], w1[1], offset_minus_4); + w1[1] = amd_bytealign (w1[1], w1[0], offset_minus_4); + w1[0] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w0[3] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w0[2] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w0[1] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w0[0] = amd_bytealign (w0[0], 0, offset_minus_4); + + if (offset_mod_4 == 0) + { + w0[0] = w0[1]; + w0[1] = w0[2]; + w0[2] = w0[3]; + w0[3] = w1[0]; + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 1: + w3[2] = amd_bytealign ( 0, w3[0], offset_minus_4); + w3[1] = amd_bytealign (w3[0], w2[3], offset_minus_4); + w3[0] = amd_bytealign (w2[3], w2[2], offset_minus_4); + w2[3] = amd_bytealign (w2[2], w2[1], offset_minus_4); + w2[2] = amd_bytealign (w2[1], w2[0], offset_minus_4); + w2[1] = amd_bytealign (w2[0], w1[3], offset_minus_4); + w2[0] = amd_bytealign (w1[3], w1[2], offset_minus_4); + w1[3] = amd_bytealign (w1[2], w1[1], offset_minus_4); + w1[2] = amd_bytealign (w1[1], w1[0], offset_minus_4); + w1[1] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w1[0] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w0[3] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w0[2] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w0[1] = amd_bytealign (w0[0], 0, offset_minus_4); + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w0[1] = w0[2]; + w0[2] = w0[3]; + w0[3] = w1[0]; + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 2: + w3[2] = amd_bytealign ( 0, w2[3], offset_minus_4); + w3[1] = amd_bytealign (w2[3], w2[2], offset_minus_4); + w3[0] = amd_bytealign (w2[2], w2[1], offset_minus_4); + w2[3] = amd_bytealign (w2[1], w2[0], offset_minus_4); + w2[2] = amd_bytealign (w2[0], w1[3], offset_minus_4); + w2[1] = amd_bytealign (w1[3], w1[2], offset_minus_4); + w2[0] = amd_bytealign (w1[2], w1[1], offset_minus_4); + w1[3] = amd_bytealign (w1[1], w1[0], offset_minus_4); + w1[2] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w1[1] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w1[0] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w0[3] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w0[2] = amd_bytealign (w0[0], 0, offset_minus_4); + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w0[2] = w0[3]; + w0[3] = w1[0]; + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 3: + w3[2] = amd_bytealign ( 0, w2[2], offset_minus_4); + w3[1] = amd_bytealign (w2[2], w2[1], offset_minus_4); + w3[0] = amd_bytealign (w2[1], w2[0], offset_minus_4); + w2[3] = amd_bytealign (w2[0], w1[3], offset_minus_4); + w2[2] = amd_bytealign (w1[3], w1[2], offset_minus_4); + w2[1] = amd_bytealign (w1[2], w1[1], offset_minus_4); + w2[0] = amd_bytealign (w1[1], w1[0], offset_minus_4); + w1[3] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w1[2] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w1[1] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w1[0] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w0[3] = amd_bytealign (w0[0], 0, offset_minus_4); + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w0[3] = w1[0]; + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 4: + w3[2] = amd_bytealign ( 0, w2[1], offset_minus_4); + w3[1] = amd_bytealign (w2[1], w2[0], offset_minus_4); + w3[0] = amd_bytealign (w2[0], w1[3], offset_minus_4); + w2[3] = amd_bytealign (w1[3], w1[2], offset_minus_4); + w2[2] = amd_bytealign (w1[2], w1[1], offset_minus_4); + w2[1] = amd_bytealign (w1[1], w1[0], offset_minus_4); + w2[0] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w1[3] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w1[2] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w1[1] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w1[0] = amd_bytealign (w0[0], 0, offset_minus_4); + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 5: + w3[2] = amd_bytealign ( 0, w2[0], offset_minus_4); + w3[1] = amd_bytealign (w2[0], w1[3], offset_minus_4); + w3[0] = amd_bytealign (w1[3], w1[2], offset_minus_4); + w2[3] = amd_bytealign (w1[2], w1[1], offset_minus_4); + w2[2] = amd_bytealign (w1[1], w1[0], offset_minus_4); + w2[1] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w2[0] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w1[3] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w1[2] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w1[1] = amd_bytealign (w0[0], 0, offset_minus_4); + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 6: + w3[2] = amd_bytealign ( 0, w1[3], offset_minus_4); + w3[1] = amd_bytealign (w1[3], w1[2], offset_minus_4); + w3[0] = amd_bytealign (w1[2], w1[1], offset_minus_4); + w2[3] = amd_bytealign (w1[1], w1[0], offset_minus_4); + w2[2] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w2[1] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w2[0] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w1[3] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w1[2] = amd_bytealign (w0[0], 0, offset_minus_4); + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 7: + w3[2] = amd_bytealign ( 0, w1[2], offset_minus_4); + w3[1] = amd_bytealign (w1[2], w1[1], offset_minus_4); + w3[0] = amd_bytealign (w1[1], w1[0], offset_minus_4); + w2[3] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w2[2] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w2[1] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w2[0] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w1[3] = amd_bytealign (w0[0], 0, offset_minus_4); + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 8: + w3[2] = amd_bytealign ( 0, w1[1], offset_minus_4); + w3[1] = amd_bytealign (w1[1], w1[0], offset_minus_4); + w3[0] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w2[3] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w2[2] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w2[1] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w2[0] = amd_bytealign (w0[0], 0, offset_minus_4); + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 9: + w3[2] = amd_bytealign ( 0, w1[0], offset_minus_4); + w3[1] = amd_bytealign (w1[0], w0[3], offset_minus_4); + w3[0] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w2[3] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w2[2] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w2[1] = amd_bytealign (w0[0], 0, offset_minus_4); + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 10: + w3[2] = amd_bytealign ( 0, w0[3], offset_minus_4); + w3[1] = amd_bytealign (w0[3], w0[2], offset_minus_4); + w3[0] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w2[3] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w2[2] = amd_bytealign (w0[0], 0, offset_minus_4); + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 11: + w3[2] = amd_bytealign ( 0, w0[2], offset_minus_4); + w3[1] = amd_bytealign (w0[2], w0[1], offset_minus_4); + w3[0] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w2[3] = amd_bytealign (w0[0], 0, offset_minus_4); + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 12: + w3[2] = amd_bytealign ( 0, w0[1], offset_minus_4); + w3[1] = amd_bytealign (w0[1], w0[0], offset_minus_4); + w3[0] = amd_bytealign (w0[0], 0, offset_minus_4); + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 13: + w3[2] = amd_bytealign ( 0, w0[0], offset_minus_4); + w3[1] = amd_bytealign (w0[0], 0, offset_minus_4); + w3[0] = 0; + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + } + #endif + + #ifdef IS_NV + const int offset_minus_4 = 4 - (offset % 4); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + switch (offset / 4) + { + case 0: + w3[1] = __byte_perm (w3[0], w3[1], selector); + w3[0] = __byte_perm (w2[3], w3[0], selector); + w2[3] = __byte_perm (w2[2], w2[3], selector); + w2[2] = __byte_perm (w2[1], w2[2], selector); + w2[1] = __byte_perm (w2[0], w2[1], selector); + w2[0] = __byte_perm (w1[3], w2[0], selector); + w1[3] = __byte_perm (w1[2], w1[3], selector); + w1[2] = __byte_perm (w1[1], w1[2], selector); + w1[1] = __byte_perm (w1[0], w1[1], selector); + w1[0] = __byte_perm (w0[3], w1[0], selector); + w0[3] = __byte_perm (w0[2], w0[3], selector); + w0[2] = __byte_perm (w0[1], w0[2], selector); + w0[1] = __byte_perm (w0[0], w0[1], selector); + w0[0] = __byte_perm ( 0, w0[0], selector); + + break; + + case 1: + w3[1] = __byte_perm (w2[3], w3[0], selector); + w3[0] = __byte_perm (w2[2], w2[3], selector); + w2[3] = __byte_perm (w2[1], w2[2], selector); + w2[2] = __byte_perm (w2[0], w2[1], selector); + w2[1] = __byte_perm (w1[3], w2[0], selector); + w2[0] = __byte_perm (w1[2], w1[3], selector); + w1[3] = __byte_perm (w1[1], w1[2], selector); + w1[2] = __byte_perm (w1[0], w1[1], selector); + w1[1] = __byte_perm (w0[3], w1[0], selector); + w1[0] = __byte_perm (w0[2], w0[3], selector); + w0[3] = __byte_perm (w0[1], w0[2], selector); + w0[2] = __byte_perm (w0[0], w0[1], selector); + w0[1] = __byte_perm ( 0, w0[0], selector); + w0[0] = 0; + + break; + + case 2: + w3[1] = __byte_perm (w2[2], w2[3], selector); + w3[0] = __byte_perm (w2[1], w2[2], selector); + w2[3] = __byte_perm (w2[0], w2[1], selector); + w2[2] = __byte_perm (w1[3], w2[0], selector); + w2[1] = __byte_perm (w1[2], w1[3], selector); + w2[0] = __byte_perm (w1[1], w1[2], selector); + w1[3] = __byte_perm (w1[0], w1[1], selector); + w1[2] = __byte_perm (w0[3], w1[0], selector); + w1[1] = __byte_perm (w0[2], w0[3], selector); + w1[0] = __byte_perm (w0[1], w0[2], selector); + w0[3] = __byte_perm (w0[0], w0[1], selector); + w0[2] = __byte_perm ( 0, w0[0], selector); + w0[1] = 0; + w0[0] = 0; + + break; + + case 3: + w3[1] = __byte_perm (w2[1], w2[2], selector); + w3[0] = __byte_perm (w2[0], w2[1], selector); + w2[3] = __byte_perm (w1[3], w2[0], selector); + w2[2] = __byte_perm (w1[2], w1[3], selector); + w2[1] = __byte_perm (w1[1], w1[2], selector); + w2[0] = __byte_perm (w1[0], w1[1], selector); + w1[3] = __byte_perm (w0[3], w1[0], selector); + w1[2] = __byte_perm (w0[2], w0[3], selector); + w1[1] = __byte_perm (w0[1], w0[2], selector); + w1[0] = __byte_perm (w0[0], w0[1], selector); + w0[3] = __byte_perm ( 0, w0[0], selector); + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 4: + w3[1] = __byte_perm (w2[0], w2[1], selector); + w3[0] = __byte_perm (w1[3], w2[0], selector); + w2[3] = __byte_perm (w1[2], w1[3], selector); + w2[2] = __byte_perm (w1[1], w1[2], selector); + w2[1] = __byte_perm (w1[0], w1[1], selector); + w2[0] = __byte_perm (w0[3], w1[0], selector); + w1[3] = __byte_perm (w0[2], w0[3], selector); + w1[2] = __byte_perm (w0[1], w0[2], selector); + w1[1] = __byte_perm (w0[0], w0[1], selector); + w1[0] = __byte_perm ( 0, w0[0], selector); + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 5: + w3[1] = __byte_perm (w1[3], w2[0], selector); + w3[0] = __byte_perm (w1[2], w1[3], selector); + w2[3] = __byte_perm (w1[1], w1[2], selector); + w2[2] = __byte_perm (w1[0], w1[1], selector); + w2[1] = __byte_perm (w0[3], w1[0], selector); + w2[0] = __byte_perm (w0[2], w0[3], selector); + w1[3] = __byte_perm (w0[1], w0[2], selector); + w1[2] = __byte_perm (w0[0], w0[1], selector); + w1[1] = __byte_perm ( 0, w0[0], selector); + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 6: + w3[1] = __byte_perm (w1[2], w1[3], selector); + w3[0] = __byte_perm (w1[1], w1[2], selector); + w2[3] = __byte_perm (w1[0], w1[1], selector); + w2[2] = __byte_perm (w0[3], w1[0], selector); + w2[1] = __byte_perm (w0[2], w0[3], selector); + w2[0] = __byte_perm (w0[1], w0[2], selector); + w1[3] = __byte_perm (w0[0], w0[1], selector); + w1[2] = __byte_perm ( 0, w0[0], selector); + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 7: + w3[1] = __byte_perm (w1[1], w1[2], selector); + w3[0] = __byte_perm (w1[0], w1[1], selector); + w2[3] = __byte_perm (w0[3], w1[0], selector); + w2[2] = __byte_perm (w0[2], w0[3], selector); + w2[1] = __byte_perm (w0[1], w0[2], selector); + w2[0] = __byte_perm (w0[0], w0[1], selector); + w1[3] = __byte_perm ( 0, w0[0], selector); + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 8: + w3[1] = __byte_perm (w1[0], w1[1], selector); + w3[0] = __byte_perm (w0[3], w1[0], selector); + w2[3] = __byte_perm (w0[2], w0[3], selector); + w2[2] = __byte_perm (w0[1], w0[2], selector); + w2[1] = __byte_perm (w0[0], w0[1], selector); + w2[0] = __byte_perm ( 0, w0[0], selector); + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 9: + w3[1] = __byte_perm (w0[3], w1[0], selector); + w3[0] = __byte_perm (w0[2], w0[3], selector); + w2[3] = __byte_perm (w0[1], w0[2], selector); + w2[2] = __byte_perm (w0[0], w0[1], selector); + w2[1] = __byte_perm ( 0, w0[0], selector); + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 10: + w3[1] = __byte_perm (w0[2], w0[3], selector); + w3[0] = __byte_perm (w0[1], w0[2], selector); + w2[3] = __byte_perm (w0[0], w0[1], selector); + w2[2] = __byte_perm ( 0, w0[0], selector); + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 11: + w3[1] = __byte_perm (w0[1], w0[2], selector); + w3[0] = __byte_perm (w0[0], w0[1], selector); + w2[3] = __byte_perm ( 0, w0[0], selector); + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 12: + w3[1] = __byte_perm (w0[0], w0[1], selector); + w3[0] = __byte_perm ( 0, w0[0], selector); + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 13: + w3[1] = __byte_perm ( 0, w0[0], selector); + w3[0] = 0; + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + } + #endif +} + +inline void switch_buffer_by_offset_be (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset) +{ + #if defined IS_AMD || defined IS_GENERIC + switch (offset / 4) + { + case 0: + w3[2] = amd_bytealign (w3[1], 0, offset); + w3[1] = amd_bytealign (w3[0], w3[1], offset); + w3[0] = amd_bytealign (w2[3], w3[0], offset); + w2[3] = amd_bytealign (w2[2], w2[3], offset); + w2[2] = amd_bytealign (w2[1], w2[2], offset); + w2[1] = amd_bytealign (w2[0], w2[1], offset); + w2[0] = amd_bytealign (w1[3], w2[0], offset); + w1[3] = amd_bytealign (w1[2], w1[3], offset); + w1[2] = amd_bytealign (w1[1], w1[2], offset); + w1[1] = amd_bytealign (w1[0], w1[1], offset); + w1[0] = amd_bytealign (w0[3], w1[0], offset); + w0[3] = amd_bytealign (w0[2], w0[3], offset); + w0[2] = amd_bytealign (w0[1], w0[2], offset); + w0[1] = amd_bytealign (w0[0], w0[1], offset); + w0[0] = amd_bytealign ( 0, w0[0], offset); + break; + + case 1: + w3[2] = amd_bytealign (w3[0], 0, offset); + w3[1] = amd_bytealign (w2[3], w3[0], offset); + w3[0] = amd_bytealign (w2[2], w2[3], offset); + w2[3] = amd_bytealign (w2[1], w2[2], offset); + w2[2] = amd_bytealign (w2[0], w2[1], offset); + w2[1] = amd_bytealign (w1[3], w2[0], offset); + w2[0] = amd_bytealign (w1[2], w1[3], offset); + w1[3] = amd_bytealign (w1[1], w1[2], offset); + w1[2] = amd_bytealign (w1[0], w1[1], offset); + w1[1] = amd_bytealign (w0[3], w1[0], offset); + w1[0] = amd_bytealign (w0[2], w0[3], offset); + w0[3] = amd_bytealign (w0[1], w0[2], offset); + w0[2] = amd_bytealign (w0[0], w0[1], offset); + w0[1] = amd_bytealign ( 0, w0[0], offset); + w0[0] = 0; + break; + + case 2: + w3[2] = amd_bytealign (w2[3], 0, offset); + w3[1] = amd_bytealign (w2[2], w2[3], offset); + w3[0] = amd_bytealign (w2[1], w2[2], offset); + w2[3] = amd_bytealign (w2[0], w2[1], offset); + w2[2] = amd_bytealign (w1[3], w2[0], offset); + w2[1] = amd_bytealign (w1[2], w1[3], offset); + w2[0] = amd_bytealign (w1[1], w1[2], offset); + w1[3] = amd_bytealign (w1[0], w1[1], offset); + w1[2] = amd_bytealign (w0[3], w1[0], offset); + w1[1] = amd_bytealign (w0[2], w0[3], offset); + w1[0] = amd_bytealign (w0[1], w0[2], offset); + w0[3] = amd_bytealign (w0[0], w0[1], offset); + w0[2] = amd_bytealign ( 0, w0[0], offset); + w0[1] = 0; + w0[0] = 0; + break; + + case 3: + w3[2] = amd_bytealign (w2[2], 0, offset); + w3[1] = amd_bytealign (w2[1], w2[2], offset); + w3[0] = amd_bytealign (w2[0], w2[1], offset); + w2[3] = amd_bytealign (w1[3], w2[0], offset); + w2[2] = amd_bytealign (w1[2], w1[3], offset); + w2[1] = amd_bytealign (w1[1], w1[2], offset); + w2[0] = amd_bytealign (w1[0], w1[1], offset); + w1[3] = amd_bytealign (w0[3], w1[0], offset); + w1[2] = amd_bytealign (w0[2], w0[3], offset); + w1[1] = amd_bytealign (w0[1], w0[2], offset); + w1[0] = amd_bytealign (w0[0], w0[1], offset); + w0[3] = amd_bytealign ( 0, w0[0], offset); + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 4: + w3[2] = amd_bytealign (w2[1], 0, offset); + w3[1] = amd_bytealign (w2[0], w2[1], offset); + w3[0] = amd_bytealign (w1[3], w2[0], offset); + w2[3] = amd_bytealign (w1[2], w1[3], offset); + w2[2] = amd_bytealign (w1[1], w1[2], offset); + w2[1] = amd_bytealign (w1[0], w1[1], offset); + w2[0] = amd_bytealign (w0[3], w1[0], offset); + w1[3] = amd_bytealign (w0[2], w0[3], offset); + w1[2] = amd_bytealign (w0[1], w0[2], offset); + w1[1] = amd_bytealign (w0[0], w0[1], offset); + w1[0] = amd_bytealign ( 0, w0[0], offset); + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 5: + w3[2] = amd_bytealign (w2[0], 0, offset); + w3[1] = amd_bytealign (w1[3], w2[0], offset); + w3[0] = amd_bytealign (w1[2], w1[3], offset); + w2[3] = amd_bytealign (w1[1], w1[2], offset); + w2[2] = amd_bytealign (w1[0], w1[1], offset); + w2[1] = amd_bytealign (w0[3], w1[0], offset); + w2[0] = amd_bytealign (w0[2], w0[3], offset); + w1[3] = amd_bytealign (w0[1], w0[2], offset); + w1[2] = amd_bytealign (w0[0], w0[1], offset); + w1[1] = amd_bytealign ( 0, w0[0], offset); + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 6: + w3[2] = amd_bytealign (w1[3], 0, offset); + w3[1] = amd_bytealign (w1[2], w1[3], offset); + w3[0] = amd_bytealign (w1[1], w1[2], offset); + w2[3] = amd_bytealign (w1[0], w1[1], offset); + w2[2] = amd_bytealign (w0[3], w1[0], offset); + w2[1] = amd_bytealign (w0[2], w0[3], offset); + w2[0] = amd_bytealign (w0[1], w0[2], offset); + w1[3] = amd_bytealign (w0[0], w0[1], offset); + w1[2] = amd_bytealign ( 0, w0[0], offset); + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 7: + w3[2] = amd_bytealign (w1[2], 0, offset); + w3[1] = amd_bytealign (w1[1], w1[2], offset); + w3[0] = amd_bytealign (w1[0], w1[1], offset); + w2[3] = amd_bytealign (w0[3], w1[0], offset); + w2[2] = amd_bytealign (w0[2], w0[3], offset); + w2[1] = amd_bytealign (w0[1], w0[2], offset); + w2[0] = amd_bytealign (w0[0], w0[1], offset); + w1[3] = amd_bytealign ( 0, w0[0], offset); + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 8: + w3[2] = amd_bytealign (w1[1], 0, offset); + w3[1] = amd_bytealign (w1[0], w1[1], offset); + w3[0] = amd_bytealign (w0[3], w1[0], offset); + w2[3] = amd_bytealign (w0[2], w0[3], offset); + w2[2] = amd_bytealign (w0[1], w0[2], offset); + w2[1] = amd_bytealign (w0[0], w0[1], offset); + w2[0] = amd_bytealign ( 0, w0[0], offset); + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 9: + w3[2] = amd_bytealign (w1[0], 0, offset); + w3[1] = amd_bytealign (w0[3], w1[0], offset); + w3[0] = amd_bytealign (w0[2], w0[3], offset); + w2[3] = amd_bytealign (w0[1], w0[2], offset); + w2[2] = amd_bytealign (w0[0], w0[1], offset); + w2[1] = amd_bytealign ( 0, w0[0], offset); + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 10: + w3[2] = amd_bytealign (w0[3], 0, offset); + w3[1] = amd_bytealign (w0[2], w0[3], offset); + w3[0] = amd_bytealign (w0[1], w0[2], offset); + w2[3] = amd_bytealign (w0[0], w0[1], offset); + w2[2] = amd_bytealign ( 0, w0[0], offset); + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 11: + w3[2] = amd_bytealign (w0[2], 0, offset); + w3[1] = amd_bytealign (w0[1], w0[2], offset); + w3[0] = amd_bytealign (w0[0], w0[1], offset); + w2[3] = amd_bytealign ( 0, w0[0], offset); + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 12: + w3[2] = amd_bytealign (w0[1], 0, offset); + w3[1] = amd_bytealign (w0[0], w0[1], offset); + w3[0] = amd_bytealign ( 0, w0[0], offset); + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 13: + w3[2] = amd_bytealign (w0[0], 0, offset); + w3[1] = amd_bytealign ( 0, w0[0], offset); + w3[0] = 0; + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + } + #endif + + #ifdef IS_NV + const int selector = (0x76543210 >> ((offset & 3) * 4)) & 0xffff; + + switch (offset / 4) + { + case 0: + w3[1] = __byte_perm (w3[1], w3[0], selector); + w3[0] = __byte_perm (w3[0], w2[3], selector); + w2[3] = __byte_perm (w2[3], w2[2], selector); + w2[2] = __byte_perm (w2[2], w2[1], selector); + w2[1] = __byte_perm (w2[1], w2[0], selector); + w2[0] = __byte_perm (w2[0], w1[3], selector); + w1[3] = __byte_perm (w1[3], w1[2], selector); + w1[2] = __byte_perm (w1[2], w1[1], selector); + w1[1] = __byte_perm (w1[1], w1[0], selector); + w1[0] = __byte_perm (w1[0], w0[3], selector); + w0[3] = __byte_perm (w0[3], w0[2], selector); + w0[2] = __byte_perm (w0[2], w0[1], selector); + w0[1] = __byte_perm (w0[1], w0[0], selector); + w0[0] = __byte_perm (w0[0], 0, selector); + break; + + case 1: + w3[1] = __byte_perm (w3[0], w2[3], selector); + w3[0] = __byte_perm (w2[3], w2[2], selector); + w2[3] = __byte_perm (w2[2], w2[1], selector); + w2[2] = __byte_perm (w2[1], w2[0], selector); + w2[1] = __byte_perm (w2[0], w1[3], selector); + w2[0] = __byte_perm (w1[3], w1[2], selector); + w1[3] = __byte_perm (w1[2], w1[1], selector); + w1[2] = __byte_perm (w1[1], w1[0], selector); + w1[1] = __byte_perm (w1[0], w0[3], selector); + w1[0] = __byte_perm (w0[3], w0[2], selector); + w0[3] = __byte_perm (w0[2], w0[1], selector); + w0[2] = __byte_perm (w0[1], w0[0], selector); + w0[1] = __byte_perm (w0[0], 0, selector); + w0[0] = 0; + break; + + case 2: + w3[1] = __byte_perm (w2[3], w2[2], selector); + w3[0] = __byte_perm (w2[2], w2[1], selector); + w2[3] = __byte_perm (w2[1], w2[0], selector); + w2[2] = __byte_perm (w2[0], w1[3], selector); + w2[1] = __byte_perm (w1[3], w1[2], selector); + w2[0] = __byte_perm (w1[2], w1[1], selector); + w1[3] = __byte_perm (w1[1], w1[0], selector); + w1[2] = __byte_perm (w1[0], w0[3], selector); + w1[1] = __byte_perm (w0[3], w0[2], selector); + w1[0] = __byte_perm (w0[2], w0[1], selector); + w0[3] = __byte_perm (w0[1], w0[0], selector); + w0[2] = __byte_perm (w0[0], 0, selector); + w0[1] = 0; + w0[0] = 0; + break; + + case 3: + w3[1] = __byte_perm (w2[2], w2[1], selector); + w3[0] = __byte_perm (w2[1], w2[0], selector); + w2[3] = __byte_perm (w2[0], w1[3], selector); + w2[2] = __byte_perm (w1[3], w1[2], selector); + w2[1] = __byte_perm (w1[2], w1[1], selector); + w2[0] = __byte_perm (w1[1], w1[0], selector); + w1[3] = __byte_perm (w1[0], w0[3], selector); + w1[2] = __byte_perm (w0[3], w0[2], selector); + w1[1] = __byte_perm (w0[2], w0[1], selector); + w1[0] = __byte_perm (w0[1], w0[0], selector); + w0[3] = __byte_perm (w0[0], 0, selector); + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 4: + w3[1] = __byte_perm (w2[1], w2[0], selector); + w3[0] = __byte_perm (w2[0], w1[3], selector); + w2[3] = __byte_perm (w1[3], w1[2], selector); + w2[2] = __byte_perm (w1[2], w1[1], selector); + w2[1] = __byte_perm (w1[1], w1[0], selector); + w2[0] = __byte_perm (w1[0], w0[3], selector); + w1[3] = __byte_perm (w0[3], w0[2], selector); + w1[2] = __byte_perm (w0[2], w0[1], selector); + w1[1] = __byte_perm (w0[1], w0[0], selector); + w1[0] = __byte_perm (w0[0], 0, selector); + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 5: + w3[1] = __byte_perm (w2[0], w1[3], selector); + w3[0] = __byte_perm (w1[3], w1[2], selector); + w2[3] = __byte_perm (w1[2], w1[1], selector); + w2[2] = __byte_perm (w1[1], w1[0], selector); + w2[1] = __byte_perm (w1[0], w0[3], selector); + w2[0] = __byte_perm (w0[3], w0[2], selector); + w1[3] = __byte_perm (w0[2], w0[1], selector); + w1[2] = __byte_perm (w0[1], w0[0], selector); + w1[1] = __byte_perm (w0[0], 0, selector); + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 6: + w3[1] = __byte_perm (w1[3], w1[2], selector); + w3[0] = __byte_perm (w1[2], w1[1], selector); + w2[3] = __byte_perm (w1[1], w1[0], selector); + w2[2] = __byte_perm (w1[0], w0[3], selector); + w2[1] = __byte_perm (w0[3], w0[2], selector); + w2[0] = __byte_perm (w0[2], w0[1], selector); + w1[3] = __byte_perm (w0[1], w0[0], selector); + w1[2] = __byte_perm (w0[0], 0, selector); + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 7: + w3[1] = __byte_perm (w1[2], w1[1], selector); + w3[0] = __byte_perm (w1[1], w1[0], selector); + w2[3] = __byte_perm (w1[0], w0[3], selector); + w2[2] = __byte_perm (w0[3], w0[2], selector); + w2[1] = __byte_perm (w0[2], w0[1], selector); + w2[0] = __byte_perm (w0[1], w0[0], selector); + w1[3] = __byte_perm (w0[0], 0, selector); + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 8: + w3[1] = __byte_perm (w1[1], w1[0], selector); + w3[0] = __byte_perm (w1[0], w0[3], selector); + w2[3] = __byte_perm (w0[3], w0[2], selector); + w2[2] = __byte_perm (w0[2], w0[1], selector); + w2[1] = __byte_perm (w0[1], w0[0], selector); + w2[0] = __byte_perm (w0[0], 0, selector); + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 9: + w3[1] = __byte_perm (w1[0], w0[3], selector); + w3[0] = __byte_perm (w0[3], w0[2], selector); + w2[3] = __byte_perm (w0[2], w0[1], selector); + w2[2] = __byte_perm (w0[1], w0[0], selector); + w2[1] = __byte_perm (w0[0], 0, selector); + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 10: + w3[1] = __byte_perm (w0[3], w0[2], selector); + w3[0] = __byte_perm (w0[2], w0[1], selector); + w2[3] = __byte_perm (w0[1], w0[0], selector); + w2[2] = __byte_perm (w0[0], 0, selector); + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 11: + w3[1] = __byte_perm (w0[2], w0[1], selector); + w3[0] = __byte_perm (w0[1], w0[0], selector); + w2[3] = __byte_perm (w0[0], 0, selector); + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 12: + w3[1] = __byte_perm (w0[1], w0[0], selector); + w3[0] = __byte_perm (w0[0], 0, selector); + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 13: + w3[1] = __byte_perm (w0[0], 0, selector); + w3[0] = 0; + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + } + #endif +} + +inline void overwrite_at_le (u32x sw[16], const u32x w0, const u32 salt_len) +{ + #if defined cl_amd_media_ops + switch (salt_len) + { + case 0: sw[0] = w0; + break; + case 1: sw[0] = amd_bytealign (w0, sw[0] << 24, 3); + sw[1] = amd_bytealign (sw[1] >> 8, w0, 3); + break; + case 2: sw[0] = amd_bytealign (w0, sw[0] << 16, 2); + sw[1] = amd_bytealign (sw[1] >> 16, w0, 2); + break; + case 3: sw[0] = amd_bytealign (w0, sw[0] << 8, 1); + sw[1] = amd_bytealign (sw[1] >> 24, w0, 1); + break; + case 4: sw[1] = w0; + break; + case 5: sw[1] = amd_bytealign (w0, sw[1] << 24, 3); + sw[2] = amd_bytealign (sw[2] >> 8, w0, 3); + break; + case 6: sw[1] = amd_bytealign (w0, sw[1] << 16, 2); + sw[2] = amd_bytealign (sw[2] >> 16, w0, 2); + break; + case 7: sw[1] = amd_bytealign (w0, sw[1] << 8, 1); + sw[2] = amd_bytealign (sw[2] >> 24, w0, 1); + break; + case 8: sw[2] = w0; + break; + case 9: sw[2] = amd_bytealign (w0, sw[2] << 24, 3); + sw[3] = amd_bytealign (sw[3] >> 8, w0, 3); + break; + case 10: sw[2] = amd_bytealign (w0, sw[2] << 16, 2); + sw[3] = amd_bytealign (sw[3] >> 16, w0, 2); + break; + case 11: sw[2] = amd_bytealign (w0, sw[2] << 8, 1); + sw[3] = amd_bytealign (sw[3] >> 24, w0, 1); + break; + case 12: sw[3] = w0; + break; + case 13: sw[3] = amd_bytealign (w0, sw[3] << 24, 3); + sw[4] = amd_bytealign (sw[4] >> 8, w0, 3); + break; + case 14: sw[3] = amd_bytealign (w0, sw[3] << 16, 2); + sw[4] = amd_bytealign (sw[4] >> 16, w0, 2); + break; + case 15: sw[3] = amd_bytealign (w0, sw[3] << 8, 1); + sw[4] = amd_bytealign (sw[4] >> 24, w0, 1); + break; + case 16: sw[4] = w0; + break; + case 17: sw[4] = amd_bytealign (w0, sw[4] << 24, 3); + sw[5] = amd_bytealign (sw[5] >> 8, w0, 3); + break; + case 18: sw[4] = amd_bytealign (w0, sw[4] << 16, 2); + sw[5] = amd_bytealign (sw[5] >> 16, w0, 2); + break; + case 19: sw[4] = amd_bytealign (w0, sw[4] << 8, 1); + sw[5] = amd_bytealign (sw[5] >> 24, w0, 1); + break; + case 20: sw[5] = w0; + break; + case 21: sw[5] = amd_bytealign (w0, sw[5] << 24, 3); + sw[6] = amd_bytealign (sw[6] >> 8, w0, 3); + break; + case 22: sw[5] = amd_bytealign (w0, sw[5] << 16, 2); + sw[6] = amd_bytealign (sw[6] >> 16, w0, 2); + break; + case 23: sw[5] = amd_bytealign (w0, sw[5] << 8, 1); + sw[6] = amd_bytealign (sw[6] >> 24, w0, 1); + break; + case 24: sw[6] = w0; + break; + case 25: sw[6] = amd_bytealign (w0, sw[6] << 24, 3); + sw[7] = amd_bytealign (sw[7] >> 8, w0, 3); + break; + case 26: sw[6] = amd_bytealign (w0, sw[6] << 16, 2); + sw[7] = amd_bytealign (sw[7] >> 16, w0, 2); + break; + case 27: sw[6] = amd_bytealign (w0, sw[6] << 8, 1); + sw[7] = amd_bytealign (sw[7] >> 24, w0, 1); + break; + case 28: sw[7] = w0; + break; + case 29: sw[7] = amd_bytealign (w0, sw[7] << 24, 3); + sw[8] = amd_bytealign (sw[8] >> 8, w0, 3); + break; + case 30: sw[7] = amd_bytealign (w0, sw[7] << 16, 2); + sw[8] = amd_bytealign (sw[8] >> 16, w0, 2); + break; + case 31: sw[7] = amd_bytealign (w0, sw[7] << 8, 1); + sw[8] = amd_bytealign (sw[8] >> 24, w0, 1); + break; + } + #else + switch (salt_len) + { + case 0: sw[0] = w0; + break; + case 1: sw[0] = (sw[0] & 0x000000ff) | (w0 << 8); + sw[1] = (sw[1] & 0xffffff00) | (w0 >> 24); + break; + case 2: sw[0] = (sw[0] & 0x0000ffff) | (w0 << 16); + sw[1] = (sw[1] & 0xffff0000) | (w0 >> 16); + break; + case 3: sw[0] = (sw[0] & 0x00ffffff) | (w0 << 24); + sw[1] = (sw[1] & 0xff000000) | (w0 >> 8); + break; + case 4: sw[1] = w0; + break; + case 5: sw[1] = (sw[1] & 0x000000ff) | (w0 << 8); + sw[2] = (sw[2] & 0xffffff00) | (w0 >> 24); + break; + case 6: sw[1] = (sw[1] & 0x0000ffff) | (w0 << 16); + sw[2] = (sw[2] & 0xffff0000) | (w0 >> 16); + break; + case 7: sw[1] = (sw[1] & 0x00ffffff) | (w0 << 24); + sw[2] = (sw[2] & 0xff000000) | (w0 >> 8); + break; + case 8: sw[2] = w0; + break; + case 9: sw[2] = (sw[2] & 0x000000ff) | (w0 << 8); + sw[3] = (sw[3] & 0xffffff00) | (w0 >> 24); + break; + case 10: sw[2] = (sw[2] & 0x0000ffff) | (w0 << 16); + sw[3] = (sw[3] & 0xffff0000) | (w0 >> 16); + break; + case 11: sw[2] = (sw[2] & 0x00ffffff) | (w0 << 24); + sw[3] = (sw[3] & 0xff000000) | (w0 >> 8); + break; + case 12: sw[3] = w0; + break; + case 13: sw[3] = (sw[3] & 0x000000ff) | (w0 << 8); + sw[4] = (sw[4] & 0xffffff00) | (w0 >> 24); + break; + case 14: sw[3] = (sw[3] & 0x0000ffff) | (w0 << 16); + sw[4] = (sw[4] & 0xffff0000) | (w0 >> 16); + break; + case 15: sw[3] = (sw[3] & 0x00ffffff) | (w0 << 24); + sw[4] = (sw[4] & 0xff000000) | (w0 >> 8); + break; + case 16: sw[4] = w0; + break; + case 17: sw[4] = (sw[4] & 0x000000ff) | (w0 << 8); + sw[5] = (sw[5] & 0xffffff00) | (w0 >> 24); + break; + case 18: sw[4] = (sw[4] & 0x0000ffff) | (w0 << 16); + sw[5] = (sw[5] & 0xffff0000) | (w0 >> 16); + break; + case 19: sw[4] = (sw[4] & 0x00ffffff) | (w0 << 24); + sw[5] = (sw[5] & 0xff000000) | (w0 >> 8); + break; + case 20: sw[5] = w0; + break; + case 21: sw[5] = (sw[5] & 0x000000ff) | (w0 << 8); + sw[6] = (sw[6] & 0xffffff00) | (w0 >> 24); + break; + case 22: sw[5] = (sw[5] & 0x0000ffff) | (w0 << 16); + sw[6] = (sw[6] & 0xffff0000) | (w0 >> 16); + break; + case 23: sw[5] = (sw[5] & 0x00ffffff) | (w0 << 24); + sw[6] = (sw[6] & 0xff000000) | (w0 >> 8); + break; + case 24: sw[6] = w0; + break; + case 25: sw[6] = (sw[6] & 0x000000ff) | (w0 << 8); + sw[7] = (sw[7] & 0xffffff00) | (w0 >> 24); + break; + case 26: sw[6] = (sw[6] & 0x0000ffff) | (w0 << 16); + sw[7] = (sw[7] & 0xffff0000) | (w0 >> 16); + break; + case 27: sw[6] = (sw[6] & 0x00ffffff) | (w0 << 24); + sw[7] = (sw[7] & 0xff000000) | (w0 >> 8); + break; + case 28: sw[7] = w0; + break; + case 29: sw[7] = (sw[7] & 0x000000ff) | (w0 << 8); + sw[8] = (sw[8] & 0xffffff00) | (w0 >> 24); + break; + case 30: sw[7] = (sw[7] & 0x0000ffff) | (w0 << 16); + sw[8] = (sw[8] & 0xffff0000) | (w0 >> 16); + break; + case 31: sw[7] = (sw[7] & 0x00ffffff) | (w0 << 24); + sw[8] = (sw[8] & 0xff000000) | (w0 >> 8); + break; + } + #endif +} + +inline void overwrite_at_be (u32x sw[16], const u32x w0, const u32 salt_len) +{ + // would be nice to have optimization based on amd_bytealign as with _le counterpart + + switch (salt_len) + { + case 0: sw[0] = w0; + break; + case 1: sw[0] = (sw[0] & 0xff000000) | (w0 >> 8); + sw[1] = (sw[1] & 0x00ffffff) | (w0 << 24); + break; + case 2: sw[0] = (sw[0] & 0xffff0000) | (w0 >> 16); + sw[1] = (sw[1] & 0x0000ffff) | (w0 << 16); + break; + case 3: sw[0] = (sw[0] & 0xffffff00) | (w0 >> 24); + sw[1] = (sw[1] & 0x000000ff) | (w0 << 8); + break; + case 4: sw[1] = w0; + break; + case 5: sw[1] = (sw[1] & 0xff000000) | (w0 >> 8); + sw[2] = (sw[2] & 0x00ffffff) | (w0 << 24); + break; + case 6: sw[1] = (sw[1] & 0xffff0000) | (w0 >> 16); + sw[2] = (sw[2] & 0x0000ffff) | (w0 << 16); + break; + case 7: sw[1] = (sw[1] & 0xffffff00) | (w0 >> 24); + sw[2] = (sw[2] & 0x000000ff) | (w0 << 8); + break; + case 8: sw[2] = w0; + break; + case 9: sw[2] = (sw[2] & 0xff000000) | (w0 >> 8); + sw[3] = (sw[3] & 0x00ffffff) | (w0 << 24); + break; + case 10: sw[2] = (sw[2] & 0xffff0000) | (w0 >> 16); + sw[3] = (sw[3] & 0x0000ffff) | (w0 << 16); + break; + case 11: sw[2] = (sw[2] & 0xffffff00) | (w0 >> 24); + sw[3] = (sw[3] & 0x000000ff) | (w0 << 8); + break; + case 12: sw[3] = w0; + break; + case 13: sw[3] = (sw[3] & 0xff000000) | (w0 >> 8); + sw[4] = (sw[4] & 0x00ffffff) | (w0 << 24); + break; + case 14: sw[3] = (sw[3] & 0xffff0000) | (w0 >> 16); + sw[4] = (sw[4] & 0x0000ffff) | (w0 << 16); + break; + case 15: sw[3] = (sw[3] & 0xffffff00) | (w0 >> 24); + sw[4] = (sw[4] & 0x000000ff) | (w0 << 8); + break; + case 16: sw[4] = w0; + break; + case 17: sw[4] = (sw[4] & 0xff000000) | (w0 >> 8); + sw[5] = (sw[5] & 0x00ffffff) | (w0 << 24); + break; + case 18: sw[4] = (sw[4] & 0xffff0000) | (w0 >> 16); + sw[5] = (sw[5] & 0x0000ffff) | (w0 << 16); + break; + case 19: sw[4] = (sw[4] & 0xffffff00) | (w0 >> 24); + sw[5] = (sw[5] & 0x000000ff) | (w0 << 8); + break; + case 20: sw[5] = w0; + break; + case 21: sw[5] = (sw[5] & 0xff000000) | (w0 >> 8); + sw[6] = (sw[6] & 0x00ffffff) | (w0 << 24); + break; + case 22: sw[5] = (sw[5] & 0xffff0000) | (w0 >> 16); + sw[6] = (sw[6] & 0x0000ffff) | (w0 << 16); + break; + case 23: sw[5] = (sw[5] & 0xffffff00) | (w0 >> 24); + sw[6] = (sw[6] & 0x000000ff) | (w0 << 8); + break; + case 24: sw[6] = w0; + break; + case 25: sw[6] = (sw[6] & 0xff000000) | (w0 >> 8); + sw[7] = (sw[7] & 0x00ffffff) | (w0 << 24); + break; + case 26: sw[6] = (sw[6] & 0xffff0000) | (w0 >> 16); + sw[7] = (sw[7] & 0x0000ffff) | (w0 << 16); + break; + case 27: sw[6] = (sw[6] & 0xffffff00) | (w0 >> 24); + sw[7] = (sw[7] & 0x000000ff) | (w0 << 8); + break; + case 28: sw[7] = w0; + break; + case 29: sw[7] = (sw[7] & 0xff000000) | (w0 >> 8); + sw[8] = (sw[8] & 0x00ffffff) | (w0 << 24); + break; + case 30: sw[7] = (sw[7] & 0xffff0000) | (w0 >> 16); + sw[8] = (sw[8] & 0x0000ffff) | (w0 << 16); + break; + case 31: sw[7] = (sw[7] & 0xffffff00) | (w0 >> 24); + sw[8] = (sw[8] & 0x000000ff) | (w0 << 8); + break; + } +} + +inline void overwrite_at_le_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x wx, const u32 salt_len) +{ + #if defined cl_amd_media_ops + switch (salt_len) + { + case 0: w0[0] = wx; + break; + case 1: w0[0] = amd_bytealign (wx, w0[0] << 24, 3); + w0[1] = amd_bytealign (w0[1] >> 8, wx, 3); + break; + case 2: w0[0] = amd_bytealign (wx, w0[0] << 16, 2); + w0[1] = amd_bytealign (w0[1] >> 16, wx, 2); + break; + case 3: w0[0] = amd_bytealign (wx, w0[0] << 8, 1); + w0[1] = amd_bytealign (w0[1] >> 24, wx, 1); + break; + case 4: w0[1] = wx; + break; + case 5: w0[1] = amd_bytealign (wx, w0[1] << 24, 3); + w0[2] = amd_bytealign (w0[2] >> 8, wx, 3); + break; + case 6: w0[1] = amd_bytealign (wx, w0[1] << 16, 2); + w0[2] = amd_bytealign (w0[2] >> 16, wx, 2); + break; + case 7: w0[1] = amd_bytealign (wx, w0[1] << 8, 1); + w0[2] = amd_bytealign (w0[2] >> 24, wx, 1); + break; + case 8: w0[2] = wx; + break; + case 9: w0[2] = amd_bytealign (wx, w0[2] << 24, 3); + w0[3] = amd_bytealign (w0[3] >> 8, wx, 3); + break; + case 10: w0[2] = amd_bytealign (wx, w0[2] << 16, 2); + w0[3] = amd_bytealign (w0[3] >> 16, wx, 2); + break; + case 11: w0[2] = amd_bytealign (wx, w0[2] << 8, 1); + w0[3] = amd_bytealign (w0[3] >> 24, wx, 1); + break; + case 12: w0[3] = wx; + break; + case 13: w0[3] = amd_bytealign (wx, w0[3] << 24, 3); + w1[0] = amd_bytealign (w1[0] >> 8, wx, 3); + break; + case 14: w0[3] = amd_bytealign (wx, w0[3] << 16, 2); + w1[0] = amd_bytealign (w1[0] >> 16, wx, 2); + break; + case 15: w0[3] = amd_bytealign (wx, w0[3] << 8, 1); + w1[0] = amd_bytealign (w1[0] >> 24, wx, 1); + break; + case 16: w1[0] = wx; + break; + case 17: w1[0] = amd_bytealign (wx, w1[0] << 24, 3); + w1[1] = amd_bytealign (w1[1] >> 8, wx, 3); + break; + case 18: w1[0] = amd_bytealign (wx, w1[0] << 16, 2); + w1[1] = amd_bytealign (w1[1] >> 16, wx, 2); + break; + case 19: w1[0] = amd_bytealign (wx, w1[0] << 8, 1); + w1[1] = amd_bytealign (w1[1] >> 24, wx, 1); + break; + case 20: w1[1] = wx; + break; + case 21: w1[1] = amd_bytealign (wx, w1[1] << 24, 3); + w1[2] = amd_bytealign (w1[2] >> 8, wx, 3); + break; + case 22: w1[1] = amd_bytealign (wx, w1[1] << 16, 2); + w1[2] = amd_bytealign (w1[2] >> 16, wx, 2); + break; + case 23: w1[1] = amd_bytealign (wx, w1[1] << 8, 1); + w1[2] = amd_bytealign (w1[2] >> 24, wx, 1); + break; + case 24: w1[2] = wx; + break; + case 25: w1[2] = amd_bytealign (wx, w1[2] << 24, 3); + w1[3] = amd_bytealign (w1[3] >> 8, wx, 3); + break; + case 26: w1[2] = amd_bytealign (wx, w1[2] << 16, 2); + w1[3] = amd_bytealign (w1[3] >> 16, wx, 2); + break; + case 27: w1[2] = amd_bytealign (wx, w1[2] << 8, 1); + w1[3] = amd_bytealign (w1[3] >> 24, wx, 1); + break; + case 28: w1[3] = wx; + break; + case 29: w1[3] = amd_bytealign (wx, w1[3] << 24, 3); + w2[0] = amd_bytealign (w2[0] >> 8, wx, 3); + break; + case 30: w1[3] = amd_bytealign (wx, w1[3] << 16, 2); + w2[0] = amd_bytealign (w2[0] >> 16, wx, 2); + break; + case 31: w1[3] = amd_bytealign (wx, w1[3] << 8, 1); + w2[0] = amd_bytealign (w2[0] >> 24, wx, 1); + break; + case 32: w2[0] = wx; + break; + case 33: w2[0] = amd_bytealign (wx, w2[0] << 24, 3); + w2[1] = amd_bytealign (w2[1] >> 8, wx, 3); + break; + case 34: w2[0] = amd_bytealign (wx, w2[0] << 16, 2); + w2[1] = amd_bytealign (w2[1] >> 16, wx, 2); + break; + case 35: w2[0] = amd_bytealign (wx, w2[0] << 8, 1); + w2[1] = amd_bytealign (w2[1] >> 24, wx, 1); + break; + case 36: w2[1] = wx; + break; + case 37: w2[1] = amd_bytealign (wx, w2[1] << 24, 3); + w2[2] = amd_bytealign (w2[2] >> 8, wx, 3); + break; + case 38: w2[1] = amd_bytealign (wx, w2[1] << 16, 2); + w2[2] = amd_bytealign (w2[2] >> 16, wx, 2); + break; + case 39: w2[1] = amd_bytealign (wx, w2[1] << 8, 1); + w2[2] = amd_bytealign (w2[2] >> 24, wx, 1); + break; + case 40: w2[2] = wx; + break; + case 41: w2[2] = amd_bytealign (wx, w2[2] << 24, 3); + w2[3] = amd_bytealign (w2[3] >> 8, wx, 3); + break; + case 42: w2[2] = amd_bytealign (wx, w2[2] << 16, 2); + w2[3] = amd_bytealign (w2[3] >> 16, wx, 2); + break; + case 43: w2[2] = amd_bytealign (wx, w2[2] << 8, 1); + w2[3] = amd_bytealign (w2[3] >> 24, wx, 1); + break; + case 44: w2[3] = wx; + break; + case 45: w2[3] = amd_bytealign (wx, w2[3] << 24, 3); + w3[0] = amd_bytealign (w3[0] >> 8, wx, 3); + break; + case 46: w2[3] = amd_bytealign (wx, w2[3] << 16, 2); + w3[0] = amd_bytealign (w3[0] >> 16, wx, 2); + break; + case 47: w2[3] = amd_bytealign (wx, w2[3] << 8, 1); + w3[0] = amd_bytealign (w3[0] >> 24, wx, 1); + break; + case 48: w3[0] = wx; + break; + case 49: w3[0] = amd_bytealign (wx, w3[0] << 24, 3); + w3[1] = amd_bytealign (w3[1] >> 8, wx, 3); + break; + case 50: w3[0] = amd_bytealign (wx, w3[0] << 16, 2); + w3[1] = amd_bytealign (w3[1] >> 16, wx, 2); + break; + case 51: w3[0] = amd_bytealign (wx, w3[0] << 8, 1); + w3[1] = amd_bytealign (w3[1] >> 24, wx, 1); + break; + case 52: w3[1] = wx; + break; + case 53: w3[1] = amd_bytealign (wx, w3[1] << 24, 3); + w3[2] = amd_bytealign (w3[2] >> 8, wx, 3); + break; + case 54: w3[1] = amd_bytealign (wx, w3[1] << 16, 2); + w3[2] = amd_bytealign (w3[2] >> 16, wx, 2); + break; + case 55: w3[1] = amd_bytealign (wx, w3[1] << 8, 1); + w3[2] = amd_bytealign (w3[2] >> 24, wx, 1); + break; + case 56: w3[2] = wx; + break; + case 57: w3[2] = amd_bytealign (wx, w3[2] << 24, 3); + w3[3] = amd_bytealign (w3[3] >> 8, wx, 3); + break; + case 58: w3[2] = amd_bytealign (wx, w3[2] << 16, 2); + w3[3] = amd_bytealign (w3[3] >> 16, wx, 2); + break; + case 59: w3[2] = amd_bytealign (wx, w3[2] << 8, 1); + w3[3] = amd_bytealign (w3[3] >> 24, wx, 1); + break; + case 60: w3[3] = wx; + break; + case 61: w3[3] = amd_bytealign (wx, w3[3] << 24, 3); + //w4[0] = amd_bytealign (w4[0] >> 8, wx, 3); + break; + case 62: w3[3] = amd_bytealign (wx, w3[3] << 16, 2); + //w4[0] = amd_bytealign (w4[0] >> 16, wx, 2); + break; + case 63: w3[3] = amd_bytealign (wx, w3[3] << 8, 1); + //w4[0] = amd_bytealign (w4[0] >> 24, wx, 1); + break; + } + #else + switch (salt_len) + { + case 0: w0[0] = wx; + break; + case 1: w0[0] = (w0[0] & 0x000000ff) | (wx << 8); + w0[1] = (w0[1] & 0xffffff00) | (wx >> 24); + break; + case 2: w0[0] = (w0[0] & 0x0000ffff) | (wx << 16); + w0[1] = (w0[1] & 0xffff0000) | (wx >> 16); + break; + case 3: w0[0] = (w0[0] & 0x00ffffff) | (wx << 24); + w0[1] = (w0[1] & 0xff000000) | (wx >> 8); + break; + case 4: w0[1] = wx; + break; + case 5: w0[1] = (w0[1] & 0x000000ff) | (wx << 8); + w0[2] = (w0[2] & 0xffffff00) | (wx >> 24); + break; + case 6: w0[1] = (w0[1] & 0x0000ffff) | (wx << 16); + w0[2] = (w0[2] & 0xffff0000) | (wx >> 16); + break; + case 7: w0[1] = (w0[1] & 0x00ffffff) | (wx << 24); + w0[2] = (w0[2] & 0xff000000) | (wx >> 8); + break; + case 8: w0[2] = wx; + break; + case 9: w0[2] = (w0[2] & 0x000000ff) | (wx << 8); + w0[3] = (w0[3] & 0xffffff00) | (wx >> 24); + break; + case 10: w0[2] = (w0[2] & 0x0000ffff) | (wx << 16); + w0[3] = (w0[3] & 0xffff0000) | (wx >> 16); + break; + case 11: w0[2] = (w0[2] & 0x00ffffff) | (wx << 24); + w0[3] = (w0[3] & 0xff000000) | (wx >> 8); + break; + case 12: w0[3] = wx; + break; + case 13: w0[3] = (w0[3] & 0x000000ff) | (wx << 8); + w1[0] = (w1[0] & 0xffffff00) | (wx >> 24); + break; + case 14: w0[3] = (w0[3] & 0x0000ffff) | (wx << 16); + w1[0] = (w1[0] & 0xffff0000) | (wx >> 16); + break; + case 15: w0[3] = (w0[3] & 0x00ffffff) | (wx << 24); + w1[0] = (w1[0] & 0xff000000) | (wx >> 8); + break; + case 16: w1[0] = wx; + break; + case 17: w1[0] = (w1[0] & 0x000000ff) | (wx << 8); + w1[1] = (w1[1] & 0xffffff00) | (wx >> 24); + break; + case 18: w1[0] = (w1[0] & 0x0000ffff) | (wx << 16); + w1[1] = (w1[1] & 0xffff0000) | (wx >> 16); + break; + case 19: w1[0] = (w1[0] & 0x00ffffff) | (wx << 24); + w1[1] = (w1[1] & 0xff000000) | (wx >> 8); + break; + case 20: w1[1] = wx; + break; + case 21: w1[1] = (w1[1] & 0x000000ff) | (wx << 8); + w1[2] = (w1[2] & 0xffffff00) | (wx >> 24); + break; + case 22: w1[1] = (w1[1] & 0x0000ffff) | (wx << 16); + w1[2] = (w1[2] & 0xffff0000) | (wx >> 16); + break; + case 23: w1[1] = (w1[1] & 0x00ffffff) | (wx << 24); + w1[2] = (w1[2] & 0xff000000) | (wx >> 8); + break; + case 24: w1[2] = wx; + break; + case 25: w1[2] = (w1[2] & 0x000000ff) | (wx << 8); + w1[3] = (w1[3] & 0xffffff00) | (wx >> 24); + break; + case 26: w1[2] = (w1[2] & 0x0000ffff) | (wx << 16); + w1[3] = (w1[3] & 0xffff0000) | (wx >> 16); + break; + case 27: w1[2] = (w1[2] & 0x00ffffff) | (wx << 24); + w1[3] = (w1[3] & 0xff000000) | (wx >> 8); + break; + case 28: w1[3] = wx; + break; + case 29: w1[3] = (w1[3] & 0x000000ff) | (wx << 8); + w2[0] = (w2[0] & 0xffffff00) | (wx >> 24); + break; + case 30: w1[3] = (w1[3] & 0x0000ffff) | (wx << 16); + w2[0] = (w2[0] & 0xffff0000) | (wx >> 16); + break; + case 31: w1[3] = (w1[3] & 0x00ffffff) | (wx << 24); + w2[0] = (w2[0] & 0xff000000) | (wx >> 8); + break; + case 32: w2[0] = wx; + break; + case 33: w2[0] = (w2[0] & 0x000000ff) | (wx << 8); + w2[1] = (w2[1] & 0xffffff00) | (wx >> 24); + break; + case 34: w2[0] = (w2[0] & 0x0000ffff) | (wx << 16); + w2[1] = (w2[1] & 0xffff0000) | (wx >> 16); + break; + case 35: w2[0] = (w2[0] & 0x00ffffff) | (wx << 24); + w2[1] = (w2[1] & 0xff000000) | (wx >> 8); + break; + case 36: w2[1] = wx; + break; + case 37: w2[1] = (w2[1] & 0x000000ff) | (wx << 8); + w2[2] = (w2[2] & 0xffffff00) | (wx >> 24); + break; + case 38: w2[1] = (w2[1] & 0x0000ffff) | (wx << 16); + w2[2] = (w2[2] & 0xffff0000) | (wx >> 16); + break; + case 39: w2[1] = (w2[1] & 0x00ffffff) | (wx << 24); + w2[2] = (w2[2] & 0xff000000) | (wx >> 8); + break; + case 40: w2[2] = wx; + break; + case 41: w2[2] = (w2[2] & 0x000000ff) | (wx << 8); + w2[3] = (w2[3] & 0xffffff00) | (wx >> 24); + break; + case 42: w2[2] = (w2[2] & 0x0000ffff) | (wx << 16); + w2[3] = (w2[3] & 0xffff0000) | (wx >> 16); + break; + case 43: w2[2] = (w2[2] & 0x00ffffff) | (wx << 24); + w2[3] = (w2[3] & 0xff000000) | (wx >> 8); + break; + case 44: w2[3] = wx; + break; + case 45: w2[3] = (w2[3] & 0x000000ff) | (wx << 8); + w3[0] = (w3[0] & 0xffffff00) | (wx >> 24); + break; + case 46: w2[3] = (w2[3] & 0x0000ffff) | (wx << 16); + w3[0] = (w3[0] & 0xffff0000) | (wx >> 16); + break; + case 47: w2[3] = (w2[3] & 0x00ffffff) | (wx << 24); + w3[0] = (w3[0] & 0xff000000) | (wx >> 8); + break; + case 48: w3[0] = wx; + break; + case 49: w3[0] = (w3[0] & 0x000000ff) | (wx << 8); + w3[1] = (w3[1] & 0xffffff00) | (wx >> 24); + break; + case 50: w3[0] = (w3[0] & 0x0000ffff) | (wx << 16); + w3[1] = (w3[1] & 0xffff0000) | (wx >> 16); + break; + case 51: w3[0] = (w3[0] & 0x00ffffff) | (wx << 24); + w3[1] = (w3[1] & 0xff000000) | (wx >> 8); + break; + case 52: w3[1] = wx; + break; + case 53: w3[1] = (w3[1] & 0x000000ff) | (wx << 8); + w3[2] = (w3[2] & 0xffffff00) | (wx >> 24); + break; + case 54: w3[1] = (w3[1] & 0x0000ffff) | (wx << 16); + w3[2] = (w3[2] & 0xffff0000) | (wx >> 16); + break; + case 55: w3[1] = (w3[1] & 0x00ffffff) | (wx << 24); + w3[2] = (w3[2] & 0xff000000) | (wx >> 8); + break; + case 56: w3[2] = wx; + break; + case 57: w3[2] = (w3[2] & 0x000000ff) | (wx << 8); + w3[3] = (w3[3] & 0xffffff00) | (wx >> 24); + break; + case 58: w3[2] = (w3[2] & 0x0000ffff) | (wx << 16); + w3[3] = (w3[3] & 0xffff0000) | (wx >> 16); + break; + case 59: w3[2] = (w3[2] & 0x00ffffff) | (wx << 24); + w3[3] = (w3[3] & 0xff000000) | (wx >> 8); + break; + case 60: w3[3] = wx; + break; + case 61: w3[3] = (w3[3] & 0x000000ff) | (wx << 8); + //w4[0] = (w4[0] & 0xffffff00) | (wx >> 24); + break; + case 62: w3[3] = (w3[3] & 0x0000ffff) | (wx << 16); + //w4[0] = (w4[0] & 0xffff0000) | (wx >> 16); + break; + case 63: w3[3] = (w3[3] & 0x00ffffff) | (wx << 24); + //w4[0] = (w4[0] & 0xff000000) | (wx >> 8); + break; + } + #endif +} + +inline void overwrite_at_be_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x wx, const u32 salt_len) +{ + // would be nice to have optimization based on amd_bytealign as with _le counterpart + + switch (salt_len) + { + case 0: w0[0] = wx; + break; + case 1: w0[0] = (w0[0] & 0xff000000) | (wx >> 8); + w0[1] = (w0[1] & 0x00ffffff) | (wx << 24); + break; + case 2: w0[0] = (w0[0] & 0xffff0000) | (wx >> 16); + w0[1] = (w0[1] & 0x0000ffff) | (wx << 16); + break; + case 3: w0[0] = (w0[0] & 0xffffff00) | (wx >> 24); + w0[1] = (w0[1] & 0x000000ff) | (wx << 8); + break; + case 4: w0[1] = wx; + break; + case 5: w0[1] = (w0[1] & 0xff000000) | (wx >> 8); + w0[2] = (w0[2] & 0x00ffffff) | (wx << 24); + break; + case 6: w0[1] = (w0[1] & 0xffff0000) | (wx >> 16); + w0[2] = (w0[2] & 0x0000ffff) | (wx << 16); + break; + case 7: w0[1] = (w0[1] & 0xffffff00) | (wx >> 24); + w0[2] = (w0[2] & 0x000000ff) | (wx << 8); + break; + case 8: w0[2] = wx; + break; + case 9: w0[2] = (w0[2] & 0xff000000) | (wx >> 8); + w0[3] = (w0[3] & 0x00ffffff) | (wx << 24); + break; + case 10: w0[2] = (w0[2] & 0xffff0000) | (wx >> 16); + w0[3] = (w0[3] & 0x0000ffff) | (wx << 16); + break; + case 11: w0[2] = (w0[2] & 0xffffff00) | (wx >> 24); + w0[3] = (w0[3] & 0x000000ff) | (wx << 8); + break; + case 12: w0[3] = wx; + break; + case 13: w0[3] = (w0[3] & 0xff000000) | (wx >> 8); + w1[0] = (w1[0] & 0x00ffffff) | (wx << 24); + break; + case 14: w0[3] = (w0[3] & 0xffff0000) | (wx >> 16); + w1[0] = (w1[0] & 0x0000ffff) | (wx << 16); + break; + case 15: w0[3] = (w0[3] & 0xffffff00) | (wx >> 24); + w1[0] = (w1[0] & 0x000000ff) | (wx << 8); + break; + case 16: w1[0] = wx; + break; + case 17: w1[0] = (w1[0] & 0xff000000) | (wx >> 8); + w1[1] = (w1[1] & 0x00ffffff) | (wx << 24); + break; + case 18: w1[0] = (w1[0] & 0xffff0000) | (wx >> 16); + w1[1] = (w1[1] & 0x0000ffff) | (wx << 16); + break; + case 19: w1[0] = (w1[0] & 0xffffff00) | (wx >> 24); + w1[1] = (w1[1] & 0x000000ff) | (wx << 8); + break; + case 20: w1[1] = wx; + break; + case 21: w1[1] = (w1[1] & 0xff000000) | (wx >> 8); + w1[2] = (w1[2] & 0x00ffffff) | (wx << 24); + break; + case 22: w1[1] = (w1[1] & 0xffff0000) | (wx >> 16); + w1[2] = (w1[2] & 0x0000ffff) | (wx << 16); + break; + case 23: w1[1] = (w1[1] & 0xffffff00) | (wx >> 24); + w1[2] = (w1[2] & 0x000000ff) | (wx << 8); + break; + case 24: w1[2] = wx; + break; + case 25: w1[2] = (w1[2] & 0xff000000) | (wx >> 8); + w1[3] = (w1[3] & 0x00ffffff) | (wx << 24); + break; + case 26: w1[2] = (w1[2] & 0xffff0000) | (wx >> 16); + w1[3] = (w1[3] & 0x0000ffff) | (wx << 16); + break; + case 27: w1[2] = (w1[2] & 0xffffff00) | (wx >> 24); + w1[3] = (w1[3] & 0x000000ff) | (wx << 8); + break; + case 28: w1[3] = wx; + break; + case 29: w1[3] = (w1[3] & 0xff000000) | (wx >> 8); + w2[0] = (w2[0] & 0x00ffffff) | (wx << 24); + break; + case 30: w1[3] = (w1[3] & 0xffff0000) | (wx >> 16); + w2[0] = (w2[0] & 0x0000ffff) | (wx << 16); + break; + case 31: w1[3] = (w1[3] & 0xffffff00) | (wx >> 24); + w2[0] = (w2[0] & 0x000000ff) | (wx << 8); + break; + case 32: w2[0] = wx; + break; + case 33: w2[0] = (w2[0] & 0xff000000) | (wx >> 8); + w2[1] = (w2[1] & 0x00ffffff) | (wx << 24); + break; + case 34: w2[0] = (w2[0] & 0xffff0000) | (wx >> 16); + w2[1] = (w2[1] & 0x0000ffff) | (wx << 16); + break; + case 35: w2[0] = (w2[0] & 0xffffff00) | (wx >> 24); + w2[1] = (w2[1] & 0x000000ff) | (wx << 8); + break; + case 36: w2[1] = wx; + break; + case 37: w2[1] = (w2[1] & 0xff000000) | (wx >> 8); + w2[2] = (w2[2] & 0x00ffffff) | (wx << 24); + break; + case 38: w2[1] = (w2[1] & 0xffff0000) | (wx >> 16); + w2[2] = (w2[2] & 0x0000ffff) | (wx << 16); + break; + case 39: w2[1] = (w2[1] & 0xffffff00) | (wx >> 24); + w2[2] = (w2[2] & 0x000000ff) | (wx << 8); + break; + case 40: w2[2] = wx; + break; + case 41: w2[2] = (w2[2] & 0xff000000) | (wx >> 8); + w2[3] = (w2[3] & 0x00ffffff) | (wx << 24); + break; + case 42: w2[2] = (w2[2] & 0xffff0000) | (wx >> 16); + w2[3] = (w2[3] & 0x0000ffff) | (wx << 16); + break; + case 43: w2[2] = (w2[2] & 0xffffff00) | (wx >> 24); + w2[3] = (w2[3] & 0x000000ff) | (wx << 8); + break; + case 44: w2[3] = wx; + break; + case 45: w2[3] = (w2[3] & 0xff000000) | (wx >> 8); + w3[0] = (w3[0] & 0x00ffffff) | (wx << 24); + break; + case 46: w2[3] = (w2[3] & 0xffff0000) | (wx >> 16); + w3[0] = (w3[0] & 0x0000ffff) | (wx << 16); + break; + case 47: w2[3] = (w2[3] & 0xffffff00) | (wx >> 24); + w3[0] = (w3[0] & 0x000000ff) | (wx << 8); + break; + case 48: w3[0] = wx; + break; + case 49: w3[0] = (w3[0] & 0xff000000) | (wx >> 8); + w3[1] = (w3[1] & 0x00ffffff) | (wx << 24); + break; + case 50: w3[0] = (w3[0] & 0xffff0000) | (wx >> 16); + w3[1] = (w3[1] & 0x0000ffff) | (wx << 16); + break; + case 51: w3[0] = (w3[0] & 0xffffff00) | (wx >> 24); + w3[1] = (w3[1] & 0x000000ff) | (wx << 8); + break; + case 52: w3[1] = wx; + break; + case 53: w3[1] = (w3[1] & 0xff000000) | (wx >> 8); + w3[2] = (w3[2] & 0x00ffffff) | (wx << 24); + break; + case 54: w3[1] = (w3[1] & 0xffff0000) | (wx >> 16); + w3[2] = (w3[2] & 0x0000ffff) | (wx << 16); + break; + case 55: w3[1] = (w3[1] & 0xffffff00) | (wx >> 24); + w3[2] = (w3[2] & 0x000000ff) | (wx << 8); + break; + case 56: w3[2] = wx; + break; + case 57: w3[2] = (w3[2] & 0xff000000) | (wx >> 8); + w3[3] = (w3[3] & 0x00ffffff) | (wx << 24); + break; + case 58: w3[2] = (w3[2] & 0xffff0000) | (wx >> 16); + w3[3] = (w3[3] & 0x0000ffff) | (wx << 16); + break; + case 59: w3[2] = (w3[2] & 0xffffff00) | (wx >> 24); + w3[3] = (w3[3] & 0x000000ff) | (wx << 8); + break; + case 60: w3[3] = wx; + break; + case 61: w3[3] = (w3[3] & 0xff000000) | (wx >> 8); + //w4[0] = (w4[0] & 0x00ffffff) | (wx << 24); + break; + case 62: w3[3] = (w3[3] & 0xffff0000) | (wx >> 16); + //w4[0] = (w4[0] & 0x0000ffff) | (wx << 16); + break; + case 63: w3[3] = (w3[3] & 0xffffff00) | (wx >> 24); + //w4[0] = (w4[0] & 0x000000ff) | (wx << 8); + break; + } +} + +/** + * vector functions as scalar (for outer loop usage) + */ + +inline void append_0x01_1x4_S (u32 w0[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x01; + break; + + case 1: + w0[0] = w0[0] | 0x0100; + break; + + case 2: + w0[0] = w0[0] | 0x010000; + break; + + case 3: + w0[0] = w0[0] | 0x01000000; + break; + + case 4: + w0[1] = 0x01; + break; + + case 5: + w0[1] = w0[1] | 0x0100; + break; + + case 6: + w0[1] = w0[1] | 0x010000; + break; + + case 7: + w0[1] = w0[1] | 0x01000000; + break; + + case 8: + w0[2] = 0x01; + break; + + case 9: + w0[2] = w0[2] | 0x0100; + break; + + case 10: + w0[2] = w0[2] | 0x010000; + break; + + case 11: + w0[2] = w0[2] | 0x01000000; + break; + + case 12: + w0[3] = 0x01; + break; + + case 13: + w0[3] = w0[3] | 0x0100; + break; + + case 14: + w0[3] = w0[3] | 0x010000; + break; + + case 15: + w0[3] = w0[3] | 0x01000000; + break; + } +} + +inline void append_0x01_2x4_S (u32 w0[4], u32 w1[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x01; + break; + + case 1: + w0[0] = w0[0] | 0x0100; + break; + + case 2: + w0[0] = w0[0] | 0x010000; + break; + + case 3: + w0[0] = w0[0] | 0x01000000; + break; + + case 4: + w0[1] = 0x01; + break; + + case 5: + w0[1] = w0[1] | 0x0100; + break; + + case 6: + w0[1] = w0[1] | 0x010000; + break; + + case 7: + w0[1] = w0[1] | 0x01000000; + break; + + case 8: + w0[2] = 0x01; + break; + + case 9: + w0[2] = w0[2] | 0x0100; + break; + + case 10: + w0[2] = w0[2] | 0x010000; + break; + + case 11: + w0[2] = w0[2] | 0x01000000; + break; + + case 12: + w0[3] = 0x01; + break; + + case 13: + w0[3] = w0[3] | 0x0100; + break; + + case 14: + w0[3] = w0[3] | 0x010000; + break; + + case 15: + w0[3] = w0[3] | 0x01000000; + break; + + case 16: + w1[0] = 0x01; + break; + + case 17: + w1[0] = w1[0] | 0x0100; + break; + + case 18: + w1[0] = w1[0] | 0x010000; + break; + + case 19: + w1[0] = w1[0] | 0x01000000; + break; + + case 20: + w1[1] = 0x01; + break; + + case 21: + w1[1] = w1[1] | 0x0100; + break; + + case 22: + w1[1] = w1[1] | 0x010000; + break; + + case 23: + w1[1] = w1[1] | 0x01000000; + break; + + case 24: + w1[2] = 0x01; + break; + + case 25: + w1[2] = w1[2] | 0x0100; + break; + + case 26: + w1[2] = w1[2] | 0x010000; + break; + + case 27: + w1[2] = w1[2] | 0x01000000; + break; + + case 28: + w1[3] = 0x01; + break; + + case 29: + w1[3] = w1[3] | 0x0100; + break; + + case 30: + w1[3] = w1[3] | 0x010000; + break; + + case 31: + w1[3] = w1[3] | 0x01000000; + break; + } +} + +inline void append_0x01_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x01; + break; + + case 1: + w0[0] = w0[0] | 0x0100; + break; + + case 2: + w0[0] = w0[0] | 0x010000; + break; + + case 3: + w0[0] = w0[0] | 0x01000000; + break; + + case 4: + w0[1] = 0x01; + break; + + case 5: + w0[1] = w0[1] | 0x0100; + break; + + case 6: + w0[1] = w0[1] | 0x010000; + break; + + case 7: + w0[1] = w0[1] | 0x01000000; + break; + + case 8: + w0[2] = 0x01; + break; + + case 9: + w0[2] = w0[2] | 0x0100; + break; + + case 10: + w0[2] = w0[2] | 0x010000; + break; + + case 11: + w0[2] = w0[2] | 0x01000000; + break; + + case 12: + w0[3] = 0x01; + break; + + case 13: + w0[3] = w0[3] | 0x0100; + break; + + case 14: + w0[3] = w0[3] | 0x010000; + break; + + case 15: + w0[3] = w0[3] | 0x01000000; + break; + + case 16: + w1[0] = 0x01; + break; + + case 17: + w1[0] = w1[0] | 0x0100; + break; + + case 18: + w1[0] = w1[0] | 0x010000; + break; + + case 19: + w1[0] = w1[0] | 0x01000000; + break; + + case 20: + w1[1] = 0x01; + break; + + case 21: + w1[1] = w1[1] | 0x0100; + break; + + case 22: + w1[1] = w1[1] | 0x010000; + break; + + case 23: + w1[1] = w1[1] | 0x01000000; + break; + + case 24: + w1[2] = 0x01; + break; + + case 25: + w1[2] = w1[2] | 0x0100; + break; + + case 26: + w1[2] = w1[2] | 0x010000; + break; + + case 27: + w1[2] = w1[2] | 0x01000000; + break; + + case 28: + w1[3] = 0x01; + break; + + case 29: + w1[3] = w1[3] | 0x0100; + break; + + case 30: + w1[3] = w1[3] | 0x010000; + break; + + case 31: + w1[3] = w1[3] | 0x01000000; + break; + + case 32: + w2[0] = 0x01; + break; + + case 33: + w2[0] = w2[0] | 0x0100; + break; + + case 34: + w2[0] = w2[0] | 0x010000; + break; + + case 35: + w2[0] = w2[0] | 0x01000000; + break; + + case 36: + w2[1] = 0x01; + break; + + case 37: + w2[1] = w2[1] | 0x0100; + break; + + case 38: + w2[1] = w2[1] | 0x010000; + break; + + case 39: + w2[1] = w2[1] | 0x01000000; + break; + + case 40: + w2[2] = 0x01; + break; + + case 41: + w2[2] = w2[2] | 0x0100; + break; + + case 42: + w2[2] = w2[2] | 0x010000; + break; + + case 43: + w2[2] = w2[2] | 0x01000000; + break; + + case 44: + w2[3] = 0x01; + break; + + case 45: + w2[3] = w2[3] | 0x0100; + break; + + case 46: + w2[3] = w2[3] | 0x010000; + break; + + case 47: + w2[3] = w2[3] | 0x01000000; + break; + } +} + +inline void append_0x01_4x4_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x01; + break; + + case 1: + w0[0] = w0[0] | 0x0100; + break; + + case 2: + w0[0] = w0[0] | 0x010000; + break; + + case 3: + w0[0] = w0[0] | 0x01000000; + break; + + case 4: + w0[1] = 0x01; + break; + + case 5: + w0[1] = w0[1] | 0x0100; + break; + + case 6: + w0[1] = w0[1] | 0x010000; + break; + + case 7: + w0[1] = w0[1] | 0x01000000; + break; + + case 8: + w0[2] = 0x01; + break; + + case 9: + w0[2] = w0[2] | 0x0100; + break; + + case 10: + w0[2] = w0[2] | 0x010000; + break; + + case 11: + w0[2] = w0[2] | 0x01000000; + break; + + case 12: + w0[3] = 0x01; + break; + + case 13: + w0[3] = w0[3] | 0x0100; + break; + + case 14: + w0[3] = w0[3] | 0x010000; + break; + + case 15: + w0[3] = w0[3] | 0x01000000; + break; + + case 16: + w1[0] = 0x01; + break; + + case 17: + w1[0] = w1[0] | 0x0100; + break; + + case 18: + w1[0] = w1[0] | 0x010000; + break; + + case 19: + w1[0] = w1[0] | 0x01000000; + break; + + case 20: + w1[1] = 0x01; + break; + + case 21: + w1[1] = w1[1] | 0x0100; + break; + + case 22: + w1[1] = w1[1] | 0x010000; + break; + + case 23: + w1[1] = w1[1] | 0x01000000; + break; + + case 24: + w1[2] = 0x01; + break; + + case 25: + w1[2] = w1[2] | 0x0100; + break; + + case 26: + w1[2] = w1[2] | 0x010000; + break; + + case 27: + w1[2] = w1[2] | 0x01000000; + break; + + case 28: + w1[3] = 0x01; + break; + + case 29: + w1[3] = w1[3] | 0x0100; + break; + + case 30: + w1[3] = w1[3] | 0x010000; + break; + + case 31: + w1[3] = w1[3] | 0x01000000; + break; + + case 32: + w2[0] = 0x01; + break; + + case 33: + w2[0] = w2[0] | 0x0100; + break; + + case 34: + w2[0] = w2[0] | 0x010000; + break; + + case 35: + w2[0] = w2[0] | 0x01000000; + break; + + case 36: + w2[1] = 0x01; + break; + + case 37: + w2[1] = w2[1] | 0x0100; + break; + + case 38: + w2[1] = w2[1] | 0x010000; + break; + + case 39: + w2[1] = w2[1] | 0x01000000; + break; + + case 40: + w2[2] = 0x01; + break; + + case 41: + w2[2] = w2[2] | 0x0100; + break; + + case 42: + w2[2] = w2[2] | 0x010000; + break; + + case 43: + w2[2] = w2[2] | 0x01000000; + break; + + case 44: + w2[3] = 0x01; + break; + + case 45: + w2[3] = w2[3] | 0x0100; + break; + + case 46: + w2[3] = w2[3] | 0x010000; + break; + + case 47: + w2[3] = w2[3] | 0x01000000; + break; + + case 48: + w3[0] = 0x01; + break; + + case 49: + w3[0] = w3[0] | 0x0100; + break; + + case 50: + w3[0] = w3[0] | 0x010000; + break; + + case 51: + w3[0] = w3[0] | 0x01000000; + break; + + case 52: + w3[1] = 0x01; + break; + + case 53: + w3[1] = w3[1] | 0x0100; + break; + + case 54: + w3[1] = w3[1] | 0x010000; + break; + + case 55: + w3[1] = w3[1] | 0x01000000; + break; + + case 56: + w3[2] = 0x01; + break; + + case 57: + w3[2] = w3[2] | 0x0100; + break; + + case 58: + w3[2] = w3[2] | 0x010000; + break; + + case 59: + w3[2] = w3[2] | 0x01000000; + break; + + case 60: + w3[3] = 0x01; + break; + + case 61: + w3[3] = w3[3] | 0x0100; + break; + + case 62: + w3[3] = w3[3] | 0x010000; + break; + + case 63: + w3[3] = w3[3] | 0x01000000; + break; + } +} + +inline void append_0x02_2x4_S (u32 w0[4], u32 w1[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x02; + break; + + case 1: + w0[0] = w0[0] | 0x0200; + break; + + case 2: + w0[0] = w0[0] | 0x020000; + break; + + case 3: + w0[0] = w0[0] | 0x02000000; + break; + + case 4: + w0[1] = 0x02; + break; + + case 5: + w0[1] = w0[1] | 0x0200; + break; + + case 6: + w0[1] = w0[1] | 0x020000; + break; + + case 7: + w0[1] = w0[1] | 0x02000000; + break; + + case 8: + w0[2] = 0x02; + break; + + case 9: + w0[2] = w0[2] | 0x0200; + break; + + case 10: + w0[2] = w0[2] | 0x020000; + break; + + case 11: + w0[2] = w0[2] | 0x02000000; + break; + + case 12: + w0[3] = 0x02; + break; + + case 13: + w0[3] = w0[3] | 0x0200; + break; + + case 14: + w0[3] = w0[3] | 0x020000; + break; + + case 15: + w0[3] = w0[3] | 0x02000000; + break; + + case 16: + w1[0] = 0x02; + break; + + case 17: + w1[0] = w1[0] | 0x0200; + break; + + case 18: + w1[0] = w1[0] | 0x020000; + break; + + case 19: + w1[0] = w1[0] | 0x02000000; + break; + + case 20: + w1[1] = 0x02; + break; + + case 21: + w1[1] = w1[1] | 0x0200; + break; + + case 22: + w1[1] = w1[1] | 0x020000; + break; + + case 23: + w1[1] = w1[1] | 0x02000000; + break; + + case 24: + w1[2] = 0x02; + break; + + case 25: + w1[2] = w1[2] | 0x0200; + break; + + case 26: + w1[2] = w1[2] | 0x020000; + break; + + case 27: + w1[2] = w1[2] | 0x02000000; + break; + + case 28: + w1[3] = 0x02; + break; + + case 29: + w1[3] = w1[3] | 0x0200; + break; + + case 30: + w1[3] = w1[3] | 0x020000; + break; + + case 31: + w1[3] = w1[3] | 0x02000000; + break; + } +} + +inline void append_0x02_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x02; + break; + + case 1: + w0[0] = w0[0] | 0x0200; + break; + + case 2: + w0[0] = w0[0] | 0x020000; + break; + + case 3: + w0[0] = w0[0] | 0x02000000; + break; + + case 4: + w0[1] = 0x02; + break; + + case 5: + w0[1] = w0[1] | 0x0200; + break; + + case 6: + w0[1] = w0[1] | 0x020000; + break; + + case 7: + w0[1] = w0[1] | 0x02000000; + break; + + case 8: + w0[2] = 0x02; + break; + + case 9: + w0[2] = w0[2] | 0x0200; + break; + + case 10: + w0[2] = w0[2] | 0x020000; + break; + + case 11: + w0[2] = w0[2] | 0x02000000; + break; + + case 12: + w0[3] = 0x02; + break; + + case 13: + w0[3] = w0[3] | 0x0200; + break; + + case 14: + w0[3] = w0[3] | 0x020000; + break; + + case 15: + w0[3] = w0[3] | 0x02000000; + break; + + case 16: + w1[0] = 0x02; + break; + + case 17: + w1[0] = w1[0] | 0x0200; + break; + + case 18: + w1[0] = w1[0] | 0x020000; + break; + + case 19: + w1[0] = w1[0] | 0x02000000; + break; + + case 20: + w1[1] = 0x02; + break; + + case 21: + w1[1] = w1[1] | 0x0200; + break; + + case 22: + w1[1] = w1[1] | 0x020000; + break; + + case 23: + w1[1] = w1[1] | 0x02000000; + break; + + case 24: + w1[2] = 0x02; + break; + + case 25: + w1[2] = w1[2] | 0x0200; + break; + + case 26: + w1[2] = w1[2] | 0x020000; + break; + + case 27: + w1[2] = w1[2] | 0x02000000; + break; + + case 28: + w1[3] = 0x02; + break; + + case 29: + w1[3] = w1[3] | 0x0200; + break; + + case 30: + w1[3] = w1[3] | 0x020000; + break; + + case 31: + w1[3] = w1[3] | 0x02000000; + break; + + case 32: + w2[0] = 0x02; + break; + + case 33: + w2[0] = w2[0] | 0x0200; + break; + + case 34: + w2[0] = w2[0] | 0x020000; + break; + + case 35: + w2[0] = w2[0] | 0x02000000; + break; + + case 36: + w2[1] = 0x02; + break; + + case 37: + w2[1] = w2[1] | 0x0200; + break; + + case 38: + w2[1] = w2[1] | 0x020000; + break; + + case 39: + w2[1] = w2[1] | 0x02000000; + break; + + case 40: + w2[2] = 0x02; + break; + + case 41: + w2[2] = w2[2] | 0x0200; + break; + + case 42: + w2[2] = w2[2] | 0x020000; + break; + + case 43: + w2[2] = w2[2] | 0x02000000; + break; + + case 44: + w2[3] = 0x02; + break; + + case 45: + w2[3] = w2[3] | 0x0200; + break; + + case 46: + w2[3] = w2[3] | 0x020000; + break; + + case 47: + w2[3] = w2[3] | 0x02000000; + break; + } +} + +inline void append_0x80_1x4_S (u32 w0[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x80; + break; + + case 1: + w0[0] = w0[0] | 0x8000; + break; + + case 2: + w0[0] = w0[0] | 0x800000; + break; + + case 3: + w0[0] = w0[0] | 0x80000000; + break; + + case 4: + w0[1] = 0x80; + break; + + case 5: + w0[1] = w0[1] | 0x8000; + break; + + case 6: + w0[1] = w0[1] | 0x800000; + break; + + case 7: + w0[1] = w0[1] | 0x80000000; + break; + + case 8: + w0[2] = 0x80; + break; + + case 9: + w0[2] = w0[2] | 0x8000; + break; + + case 10: + w0[2] = w0[2] | 0x800000; + break; + + case 11: + w0[2] = w0[2] | 0x80000000; + break; + + case 12: + w0[3] = 0x80; + break; + + case 13: + w0[3] = w0[3] | 0x8000; + break; + + case 14: + w0[3] = w0[3] | 0x800000; + break; + + case 15: + w0[3] = w0[3] | 0x80000000; + break; + } +} + +inline void append_0x80_2x4_S (u32 w0[4], u32 w1[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x80; + break; + + case 1: + w0[0] = w0[0] | 0x8000; + break; + + case 2: + w0[0] = w0[0] | 0x800000; + break; + + case 3: + w0[0] = w0[0] | 0x80000000; + break; + + case 4: + w0[1] = 0x80; + break; + + case 5: + w0[1] = w0[1] | 0x8000; + break; + + case 6: + w0[1] = w0[1] | 0x800000; + break; + + case 7: + w0[1] = w0[1] | 0x80000000; + break; + + case 8: + w0[2] = 0x80; + break; + + case 9: + w0[2] = w0[2] | 0x8000; + break; + + case 10: + w0[2] = w0[2] | 0x800000; + break; + + case 11: + w0[2] = w0[2] | 0x80000000; + break; + + case 12: + w0[3] = 0x80; + break; + + case 13: + w0[3] = w0[3] | 0x8000; + break; + + case 14: + w0[3] = w0[3] | 0x800000; + break; + + case 15: + w0[3] = w0[3] | 0x80000000; + break; + + case 16: + w1[0] = 0x80; + break; + + case 17: + w1[0] = w1[0] | 0x8000; + break; + + case 18: + w1[0] = w1[0] | 0x800000; + break; + + case 19: + w1[0] = w1[0] | 0x80000000; + break; + + case 20: + w1[1] = 0x80; + break; + + case 21: + w1[1] = w1[1] | 0x8000; + break; + + case 22: + w1[1] = w1[1] | 0x800000; + break; + + case 23: + w1[1] = w1[1] | 0x80000000; + break; + + case 24: + w1[2] = 0x80; + break; + + case 25: + w1[2] = w1[2] | 0x8000; + break; + + case 26: + w1[2] = w1[2] | 0x800000; + break; + + case 27: + w1[2] = w1[2] | 0x80000000; + break; + + case 28: + w1[3] = 0x80; + break; + + case 29: + w1[3] = w1[3] | 0x8000; + break; + + case 30: + w1[3] = w1[3] | 0x800000; + break; + + case 31: + w1[3] = w1[3] | 0x80000000; + break; + } +} + +inline void append_0x80_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x80; + break; + + case 1: + w0[0] = w0[0] | 0x8000; + break; + + case 2: + w0[0] = w0[0] | 0x800000; + break; + + case 3: + w0[0] = w0[0] | 0x80000000; + break; + + case 4: + w0[1] = 0x80; + break; + + case 5: + w0[1] = w0[1] | 0x8000; + break; + + case 6: + w0[1] = w0[1] | 0x800000; + break; + + case 7: + w0[1] = w0[1] | 0x80000000; + break; + + case 8: + w0[2] = 0x80; + break; + + case 9: + w0[2] = w0[2] | 0x8000; + break; + + case 10: + w0[2] = w0[2] | 0x800000; + break; + + case 11: + w0[2] = w0[2] | 0x80000000; + break; + + case 12: + w0[3] = 0x80; + break; + + case 13: + w0[3] = w0[3] | 0x8000; + break; + + case 14: + w0[3] = w0[3] | 0x800000; + break; + + case 15: + w0[3] = w0[3] | 0x80000000; + break; + + case 16: + w1[0] = 0x80; + break; + + case 17: + w1[0] = w1[0] | 0x8000; + break; + + case 18: + w1[0] = w1[0] | 0x800000; + break; + + case 19: + w1[0] = w1[0] | 0x80000000; + break; + + case 20: + w1[1] = 0x80; + break; + + case 21: + w1[1] = w1[1] | 0x8000; + break; + + case 22: + w1[1] = w1[1] | 0x800000; + break; + + case 23: + w1[1] = w1[1] | 0x80000000; + break; + + case 24: + w1[2] = 0x80; + break; + + case 25: + w1[2] = w1[2] | 0x8000; + break; + + case 26: + w1[2] = w1[2] | 0x800000; + break; + + case 27: + w1[2] = w1[2] | 0x80000000; + break; + + case 28: + w1[3] = 0x80; + break; + + case 29: + w1[3] = w1[3] | 0x8000; + break; + + case 30: + w1[3] = w1[3] | 0x800000; + break; + + case 31: + w1[3] = w1[3] | 0x80000000; + break; + + case 32: + w2[0] = 0x80; + break; + + case 33: + w2[0] = w2[0] | 0x8000; + break; + + case 34: + w2[0] = w2[0] | 0x800000; + break; + + case 35: + w2[0] = w2[0] | 0x80000000; + break; + + case 36: + w2[1] = 0x80; + break; + + case 37: + w2[1] = w2[1] | 0x8000; + break; + + case 38: + w2[1] = w2[1] | 0x800000; + break; + + case 39: + w2[1] = w2[1] | 0x80000000; + break; + + case 40: + w2[2] = 0x80; + break; + + case 41: + w2[2] = w2[2] | 0x8000; + break; + + case 42: + w2[2] = w2[2] | 0x800000; + break; + + case 43: + w2[2] = w2[2] | 0x80000000; + break; + + case 44: + w2[3] = 0x80; + break; + + case 45: + w2[3] = w2[3] | 0x8000; + break; + + case 46: + w2[3] = w2[3] | 0x800000; + break; + + case 47: + w2[3] = w2[3] | 0x80000000; + break; + } +} + +inline void append_0x80_4x4_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset) +{ + switch (offset) + { + case 0: + w0[0] = 0x80; + break; + + case 1: + w0[0] = w0[0] | 0x8000; + break; + + case 2: + w0[0] = w0[0] | 0x800000; + break; + + case 3: + w0[0] = w0[0] | 0x80000000; + break; + + case 4: + w0[1] = 0x80; + break; + + case 5: + w0[1] = w0[1] | 0x8000; + break; + + case 6: + w0[1] = w0[1] | 0x800000; + break; + + case 7: + w0[1] = w0[1] | 0x80000000; + break; + + case 8: + w0[2] = 0x80; + break; + + case 9: + w0[2] = w0[2] | 0x8000; + break; + + case 10: + w0[2] = w0[2] | 0x800000; + break; + + case 11: + w0[2] = w0[2] | 0x80000000; + break; + + case 12: + w0[3] = 0x80; + break; + + case 13: + w0[3] = w0[3] | 0x8000; + break; + + case 14: + w0[3] = w0[3] | 0x800000; + break; + + case 15: + w0[3] = w0[3] | 0x80000000; + break; + + case 16: + w1[0] = 0x80; + break; + + case 17: + w1[0] = w1[0] | 0x8000; + break; + + case 18: + w1[0] = w1[0] | 0x800000; + break; + + case 19: + w1[0] = w1[0] | 0x80000000; + break; + + case 20: + w1[1] = 0x80; + break; + + case 21: + w1[1] = w1[1] | 0x8000; + break; + + case 22: + w1[1] = w1[1] | 0x800000; + break; + + case 23: + w1[1] = w1[1] | 0x80000000; + break; + + case 24: + w1[2] = 0x80; + break; + + case 25: + w1[2] = w1[2] | 0x8000; + break; + + case 26: + w1[2] = w1[2] | 0x800000; + break; + + case 27: + w1[2] = w1[2] | 0x80000000; + break; + + case 28: + w1[3] = 0x80; + break; + + case 29: + w1[3] = w1[3] | 0x8000; + break; + + case 30: + w1[3] = w1[3] | 0x800000; + break; + + case 31: + w1[3] = w1[3] | 0x80000000; + break; + + case 32: + w2[0] = 0x80; + break; + + case 33: + w2[0] = w2[0] | 0x8000; + break; + + case 34: + w2[0] = w2[0] | 0x800000; + break; + + case 35: + w2[0] = w2[0] | 0x80000000; + break; + + case 36: + w2[1] = 0x80; + break; + + case 37: + w2[1] = w2[1] | 0x8000; + break; + + case 38: + w2[1] = w2[1] | 0x800000; + break; + + case 39: + w2[1] = w2[1] | 0x80000000; + break; + + case 40: + w2[2] = 0x80; + break; + + case 41: + w2[2] = w2[2] | 0x8000; + break; + + case 42: + w2[2] = w2[2] | 0x800000; + break; + + case 43: + w2[2] = w2[2] | 0x80000000; + break; + + case 44: + w2[3] = 0x80; + break; + + case 45: + w2[3] = w2[3] | 0x8000; + break; + + case 46: + w2[3] = w2[3] | 0x800000; + break; + + case 47: + w2[3] = w2[3] | 0x80000000; + break; + + case 48: + w3[0] = 0x80; + break; + + case 49: + w3[0] = w3[0] | 0x8000; + break; + + case 50: + w3[0] = w3[0] | 0x800000; + break; + + case 51: + w3[0] = w3[0] | 0x80000000; + break; + + case 52: + w3[1] = 0x80; + break; + + case 53: + w3[1] = w3[1] | 0x8000; + break; + + case 54: + w3[1] = w3[1] | 0x800000; + break; + + case 55: + w3[1] = w3[1] | 0x80000000; + break; + + case 56: + w3[2] = 0x80; + break; + + case 57: + w3[2] = w3[2] | 0x8000; + break; + + case 58: + w3[2] = w3[2] | 0x800000; + break; + + case 59: + w3[2] = w3[2] | 0x80000000; + break; + + case 60: + w3[3] = 0x80; + break; + + case 61: + w3[3] = w3[3] | 0x8000; + break; + + case 62: + w3[3] = w3[3] | 0x800000; + break; + + case 63: + w3[3] = w3[3] | 0x80000000; + break; + } +} + +inline void truncate_block_S (u32 w[4], const u32 len) +{ + switch (len) + { + case 0: w[0] &= 0; + w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 1: w[0] &= 0x000000FF; + w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 2: w[0] &= 0x0000FFFF; + w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 3: w[0] &= 0x00FFFFFF; + w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 4: w[1] &= 0; + w[2] &= 0; + w[3] &= 0; + break; + case 5: w[1] &= 0x000000FF; + w[2] &= 0; + w[3] &= 0; + break; + case 6: w[1] &= 0x0000FFFF; + w[2] &= 0; + w[3] &= 0; + break; + case 7: w[1] &= 0x00FFFFFF; + w[2] &= 0; + w[3] &= 0; + break; + case 8: w[2] &= 0; + w[3] &= 0; + break; + case 9: w[2] &= 0x000000FF; + w[3] &= 0; + break; + case 10: w[2] &= 0x0000FFFF; + w[3] &= 0; + break; + case 11: w[2] &= 0x00FFFFFF; + w[3] &= 0; + break; + case 12: w[3] &= 0; + break; + case 13: w[3] &= 0x000000FF; + break; + case 14: w[3] &= 0x0000FFFF; + break; + case 15: w[3] &= 0x00FFFFFF; + break; + } +} + +inline void make_unicode_S (const u32 in[4], u32 out1[4], u32 out2[4]) +{ + #ifdef IS_NV + out2[3] = __byte_perm_S (in[3], 0, 0x7372); + out2[2] = __byte_perm_S (in[3], 0, 0x7170); + out2[1] = __byte_perm_S (in[2], 0, 0x7372); + out2[0] = __byte_perm_S (in[2], 0, 0x7170); + out1[3] = __byte_perm_S (in[1], 0, 0x7372); + out1[2] = __byte_perm_S (in[1], 0, 0x7170); + out1[1] = __byte_perm_S (in[0], 0, 0x7372); + out1[0] = __byte_perm_S (in[0], 0, 0x7170); + #endif + + #if defined IS_AMD || defined IS_GENERIC + out2[3] = ((in[3] >> 8) & 0x00FF0000) | ((in[3] >> 16) & 0x000000FF); + out2[2] = ((in[3] << 8) & 0x00FF0000) | ((in[3] >> 0) & 0x000000FF); + out2[1] = ((in[2] >> 8) & 0x00FF0000) | ((in[2] >> 16) & 0x000000FF); + out2[0] = ((in[2] << 8) & 0x00FF0000) | ((in[2] >> 0) & 0x000000FF); + out1[3] = ((in[1] >> 8) & 0x00FF0000) | ((in[1] >> 16) & 0x000000FF); + out1[2] = ((in[1] << 8) & 0x00FF0000) | ((in[1] >> 0) & 0x000000FF); + out1[1] = ((in[0] >> 8) & 0x00FF0000) | ((in[0] >> 16) & 0x000000FF); + out1[0] = ((in[0] << 8) & 0x00FF0000) | ((in[0] >> 0) & 0x000000FF); + #endif +} + +inline void undo_unicode_S (const u32 in1[4], const u32 in2[4], u32 out[4]) +{ + #ifdef IS_NV + out[0] = __byte_perm_S (in1[0], in1[1], 0x6420); + out[1] = __byte_perm_S (in1[2], in1[3], 0x6420); + out[2] = __byte_perm_S (in2[0], in2[1], 0x6420); + out[3] = __byte_perm_S (in2[2], in2[3], 0x6420); + #endif + + #if defined IS_AMD || defined IS_GENERIC + out[0] = ((in1[0] & 0x000000ff) >> 0) | ((in1[0] & 0x00ff0000) >> 8) + | ((in1[1] & 0x000000ff) << 16) | ((in1[1] & 0x00ff0000) << 8); + out[1] = ((in1[2] & 0x000000ff) >> 0) | ((in1[2] & 0x00ff0000) >> 8) + | ((in1[3] & 0x000000ff) << 16) | ((in1[3] & 0x00ff0000) << 8); + out[2] = ((in2[0] & 0x000000ff) >> 0) | ((in2[0] & 0x00ff0000) >> 8) + | ((in2[1] & 0x000000ff) << 16) | ((in2[1] & 0x00ff0000) << 8); + out[3] = ((in2[2] & 0x000000ff) >> 0) | ((in2[2] & 0x00ff0000) >> 8) + | ((in2[3] & 0x000000ff) << 16) | ((in2[3] & 0x00ff0000) << 8); + #endif +} + +inline void switch_buffer_by_offset_le_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset) +{ + #if defined IS_AMD || defined IS_GENERIC + const int offset_mod_4 = offset & 3; + + const int offset_minus_4 = 4 - offset; + + switch (offset / 4) + { + case 0: + w3[2] = amd_bytealign_S ( 0, w3[1], offset_minus_4); + w3[1] = amd_bytealign_S (w3[1], w3[0], offset_minus_4); + w3[0] = amd_bytealign_S (w3[0], w2[3], offset_minus_4); + w2[3] = amd_bytealign_S (w2[3], w2[2], offset_minus_4); + w2[2] = amd_bytealign_S (w2[2], w2[1], offset_minus_4); + w2[1] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); + w2[0] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); + w1[3] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); + w1[2] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); + w1[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); + w1[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w0[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w0[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w0[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w0[0] = amd_bytealign_S (w0[0], 0, offset_minus_4); + + if (offset_mod_4 == 0) + { + w0[0] = w0[1]; + w0[1] = w0[2]; + w0[2] = w0[3]; + w0[3] = w1[0]; + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 1: + w3[2] = amd_bytealign_S ( 0, w3[0], offset_minus_4); + w3[1] = amd_bytealign_S (w3[0], w2[3], offset_minus_4); + w3[0] = amd_bytealign_S (w2[3], w2[2], offset_minus_4); + w2[3] = amd_bytealign_S (w2[2], w2[1], offset_minus_4); + w2[2] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); + w2[1] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); + w2[0] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); + w1[3] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); + w1[2] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); + w1[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w1[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w0[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w0[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w0[1] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w0[1] = w0[2]; + w0[2] = w0[3]; + w0[3] = w1[0]; + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 2: + w3[2] = amd_bytealign_S ( 0, w2[3], offset_minus_4); + w3[1] = amd_bytealign_S (w2[3], w2[2], offset_minus_4); + w3[0] = amd_bytealign_S (w2[2], w2[1], offset_minus_4); + w2[3] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); + w2[2] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); + w2[1] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); + w2[0] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); + w1[3] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); + w1[2] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w1[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w1[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w0[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w0[2] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w0[2] = w0[3]; + w0[3] = w1[0]; + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 3: + w3[2] = amd_bytealign_S ( 0, w2[2], offset_minus_4); + w3[1] = amd_bytealign_S (w2[2], w2[1], offset_minus_4); + w3[0] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); + w2[3] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); + w2[2] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); + w2[1] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); + w2[0] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); + w1[3] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w1[2] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w1[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w1[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w0[3] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w0[3] = w1[0]; + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 4: + w3[2] = amd_bytealign_S ( 0, w2[1], offset_minus_4); + w3[1] = amd_bytealign_S (w2[1], w2[0], offset_minus_4); + w3[0] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); + w2[3] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); + w2[2] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); + w2[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); + w2[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w1[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w1[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w1[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w1[0] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w1[0] = w1[1]; + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 5: + w3[2] = amd_bytealign_S ( 0, w2[0], offset_minus_4); + w3[1] = amd_bytealign_S (w2[0], w1[3], offset_minus_4); + w3[0] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); + w2[3] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); + w2[2] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); + w2[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w2[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w1[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w1[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w1[1] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w1[1] = w1[2]; + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 6: + w3[2] = amd_bytealign_S ( 0, w1[3], offset_minus_4); + w3[1] = amd_bytealign_S (w1[3], w1[2], offset_minus_4); + w3[0] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); + w2[3] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); + w2[2] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w2[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w2[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w1[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w1[2] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w1[2] = w1[3]; + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 7: + w3[2] = amd_bytealign_S ( 0, w1[2], offset_minus_4); + w3[1] = amd_bytealign_S (w1[2], w1[1], offset_minus_4); + w3[0] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); + w2[3] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w2[2] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w2[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w2[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w1[3] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w1[3] = w2[0]; + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 8: + w3[2] = amd_bytealign_S ( 0, w1[1], offset_minus_4); + w3[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4); + w3[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w2[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w2[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w2[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w2[0] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w2[0] = w2[1]; + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 9: + w3[2] = amd_bytealign_S ( 0, w1[0], offset_minus_4); + w3[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4); + w3[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w2[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w2[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w2[1] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w2[1] = w2[2]; + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 10: + w3[2] = amd_bytealign_S ( 0, w0[3], offset_minus_4); + w3[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4); + w3[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w2[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w2[2] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w2[2] = w2[3]; + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 11: + w3[2] = amd_bytealign_S ( 0, w0[2], offset_minus_4); + w3[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4); + w3[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w2[3] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w2[3] = w3[0]; + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 12: + w3[2] = amd_bytealign_S ( 0, w0[1], offset_minus_4); + w3[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4); + w3[0] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w3[0] = w3[1]; + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + + case 13: + w3[2] = amd_bytealign_S ( 0, w0[0], offset_minus_4); + w3[1] = amd_bytealign_S (w0[0], 0, offset_minus_4); + w3[0] = 0; + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + if (offset_mod_4 == 0) + { + w3[1] = w3[2]; + w3[2] = 0; + } + + break; + } + #endif + + #ifdef IS_NV + const int offset_minus_4 = 4 - (offset % 4); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + switch (offset / 4) + { + case 0: + w3[1] = __byte_perm_S (w3[0], w3[1], selector); + w3[0] = __byte_perm_S (w2[3], w3[0], selector); + w2[3] = __byte_perm_S (w2[2], w2[3], selector); + w2[2] = __byte_perm_S (w2[1], w2[2], selector); + w2[1] = __byte_perm_S (w2[0], w2[1], selector); + w2[0] = __byte_perm_S (w1[3], w2[0], selector); + w1[3] = __byte_perm_S (w1[2], w1[3], selector); + w1[2] = __byte_perm_S (w1[1], w1[2], selector); + w1[1] = __byte_perm_S (w1[0], w1[1], selector); + w1[0] = __byte_perm_S (w0[3], w1[0], selector); + w0[3] = __byte_perm_S (w0[2], w0[3], selector); + w0[2] = __byte_perm_S (w0[1], w0[2], selector); + w0[1] = __byte_perm_S (w0[0], w0[1], selector); + w0[0] = __byte_perm_S ( 0, w0[0], selector); + + break; + + case 1: + w3[1] = __byte_perm_S (w2[3], w3[0], selector); + w3[0] = __byte_perm_S (w2[2], w2[3], selector); + w2[3] = __byte_perm_S (w2[1], w2[2], selector); + w2[2] = __byte_perm_S (w2[0], w2[1], selector); + w2[1] = __byte_perm_S (w1[3], w2[0], selector); + w2[0] = __byte_perm_S (w1[2], w1[3], selector); + w1[3] = __byte_perm_S (w1[1], w1[2], selector); + w1[2] = __byte_perm_S (w1[0], w1[1], selector); + w1[1] = __byte_perm_S (w0[3], w1[0], selector); + w1[0] = __byte_perm_S (w0[2], w0[3], selector); + w0[3] = __byte_perm_S (w0[1], w0[2], selector); + w0[2] = __byte_perm_S (w0[0], w0[1], selector); + w0[1] = __byte_perm_S ( 0, w0[0], selector); + w0[0] = 0; + + break; + + case 2: + w3[1] = __byte_perm_S (w2[2], w2[3], selector); + w3[0] = __byte_perm_S (w2[1], w2[2], selector); + w2[3] = __byte_perm_S (w2[0], w2[1], selector); + w2[2] = __byte_perm_S (w1[3], w2[0], selector); + w2[1] = __byte_perm_S (w1[2], w1[3], selector); + w2[0] = __byte_perm_S (w1[1], w1[2], selector); + w1[3] = __byte_perm_S (w1[0], w1[1], selector); + w1[2] = __byte_perm_S (w0[3], w1[0], selector); + w1[1] = __byte_perm_S (w0[2], w0[3], selector); + w1[0] = __byte_perm_S (w0[1], w0[2], selector); + w0[3] = __byte_perm_S (w0[0], w0[1], selector); + w0[2] = __byte_perm_S ( 0, w0[0], selector); + w0[1] = 0; + w0[0] = 0; + + break; + + case 3: + w3[1] = __byte_perm_S (w2[1], w2[2], selector); + w3[0] = __byte_perm_S (w2[0], w2[1], selector); + w2[3] = __byte_perm_S (w1[3], w2[0], selector); + w2[2] = __byte_perm_S (w1[2], w1[3], selector); + w2[1] = __byte_perm_S (w1[1], w1[2], selector); + w2[0] = __byte_perm_S (w1[0], w1[1], selector); + w1[3] = __byte_perm_S (w0[3], w1[0], selector); + w1[2] = __byte_perm_S (w0[2], w0[3], selector); + w1[1] = __byte_perm_S (w0[1], w0[2], selector); + w1[0] = __byte_perm_S (w0[0], w0[1], selector); + w0[3] = __byte_perm_S ( 0, w0[0], selector); + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 4: + w3[1] = __byte_perm_S (w2[0], w2[1], selector); + w3[0] = __byte_perm_S (w1[3], w2[0], selector); + w2[3] = __byte_perm_S (w1[2], w1[3], selector); + w2[2] = __byte_perm_S (w1[1], w1[2], selector); + w2[1] = __byte_perm_S (w1[0], w1[1], selector); + w2[0] = __byte_perm_S (w0[3], w1[0], selector); + w1[3] = __byte_perm_S (w0[2], w0[3], selector); + w1[2] = __byte_perm_S (w0[1], w0[2], selector); + w1[1] = __byte_perm_S (w0[0], w0[1], selector); + w1[0] = __byte_perm_S ( 0, w0[0], selector); + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 5: + w3[1] = __byte_perm_S (w1[3], w2[0], selector); + w3[0] = __byte_perm_S (w1[2], w1[3], selector); + w2[3] = __byte_perm_S (w1[1], w1[2], selector); + w2[2] = __byte_perm_S (w1[0], w1[1], selector); + w2[1] = __byte_perm_S (w0[3], w1[0], selector); + w2[0] = __byte_perm_S (w0[2], w0[3], selector); + w1[3] = __byte_perm_S (w0[1], w0[2], selector); + w1[2] = __byte_perm_S (w0[0], w0[1], selector); + w1[1] = __byte_perm_S ( 0, w0[0], selector); + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 6: + w3[1] = __byte_perm_S (w1[2], w1[3], selector); + w3[0] = __byte_perm_S (w1[1], w1[2], selector); + w2[3] = __byte_perm_S (w1[0], w1[1], selector); + w2[2] = __byte_perm_S (w0[3], w1[0], selector); + w2[1] = __byte_perm_S (w0[2], w0[3], selector); + w2[0] = __byte_perm_S (w0[1], w0[2], selector); + w1[3] = __byte_perm_S (w0[0], w0[1], selector); + w1[2] = __byte_perm_S ( 0, w0[0], selector); + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 7: + w3[1] = __byte_perm_S (w1[1], w1[2], selector); + w3[0] = __byte_perm_S (w1[0], w1[1], selector); + w2[3] = __byte_perm_S (w0[3], w1[0], selector); + w2[2] = __byte_perm_S (w0[2], w0[3], selector); + w2[1] = __byte_perm_S (w0[1], w0[2], selector); + w2[0] = __byte_perm_S (w0[0], w0[1], selector); + w1[3] = __byte_perm_S ( 0, w0[0], selector); + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 8: + w3[1] = __byte_perm_S (w1[0], w1[1], selector); + w3[0] = __byte_perm_S (w0[3], w1[0], selector); + w2[3] = __byte_perm_S (w0[2], w0[3], selector); + w2[2] = __byte_perm_S (w0[1], w0[2], selector); + w2[1] = __byte_perm_S (w0[0], w0[1], selector); + w2[0] = __byte_perm_S ( 0, w0[0], selector); + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 9: + w3[1] = __byte_perm_S (w0[3], w1[0], selector); + w3[0] = __byte_perm_S (w0[2], w0[3], selector); + w2[3] = __byte_perm_S (w0[1], w0[2], selector); + w2[2] = __byte_perm_S (w0[0], w0[1], selector); + w2[1] = __byte_perm_S ( 0, w0[0], selector); + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 10: + w3[1] = __byte_perm_S (w0[2], w0[3], selector); + w3[0] = __byte_perm_S (w0[1], w0[2], selector); + w2[3] = __byte_perm_S (w0[0], w0[1], selector); + w2[2] = __byte_perm_S ( 0, w0[0], selector); + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 11: + w3[1] = __byte_perm_S (w0[1], w0[2], selector); + w3[0] = __byte_perm_S (w0[0], w0[1], selector); + w2[3] = __byte_perm_S ( 0, w0[0], selector); + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 12: + w3[1] = __byte_perm_S (w0[0], w0[1], selector); + w3[0] = __byte_perm_S ( 0, w0[0], selector); + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + + case 13: + w3[1] = __byte_perm_S ( 0, w0[0], selector); + w3[0] = 0; + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + + break; + } + #endif +} + +inline void switch_buffer_by_offset_be_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset) +{ + #if defined IS_AMD || defined IS_GENERIC + switch (offset / 4) + { + case 0: + w3[2] = amd_bytealign_S (w3[1], 0, offset); + w3[1] = amd_bytealign_S (w3[0], w3[1], offset); + w3[0] = amd_bytealign_S (w2[3], w3[0], offset); + w2[3] = amd_bytealign_S (w2[2], w2[3], offset); + w2[2] = amd_bytealign_S (w2[1], w2[2], offset); + w2[1] = amd_bytealign_S (w2[0], w2[1], offset); + w2[0] = amd_bytealign_S (w1[3], w2[0], offset); + w1[3] = amd_bytealign_S (w1[2], w1[3], offset); + w1[2] = amd_bytealign_S (w1[1], w1[2], offset); + w1[1] = amd_bytealign_S (w1[0], w1[1], offset); + w1[0] = amd_bytealign_S (w0[3], w1[0], offset); + w0[3] = amd_bytealign_S (w0[2], w0[3], offset); + w0[2] = amd_bytealign_S (w0[1], w0[2], offset); + w0[1] = amd_bytealign_S (w0[0], w0[1], offset); + w0[0] = amd_bytealign_S ( 0, w0[0], offset); + break; + + case 1: + w3[2] = amd_bytealign_S (w3[0], 0, offset); + w3[1] = amd_bytealign_S (w2[3], w3[0], offset); + w3[0] = amd_bytealign_S (w2[2], w2[3], offset); + w2[3] = amd_bytealign_S (w2[1], w2[2], offset); + w2[2] = amd_bytealign_S (w2[0], w2[1], offset); + w2[1] = amd_bytealign_S (w1[3], w2[0], offset); + w2[0] = amd_bytealign_S (w1[2], w1[3], offset); + w1[3] = amd_bytealign_S (w1[1], w1[2], offset); + w1[2] = amd_bytealign_S (w1[0], w1[1], offset); + w1[1] = amd_bytealign_S (w0[3], w1[0], offset); + w1[0] = amd_bytealign_S (w0[2], w0[3], offset); + w0[3] = amd_bytealign_S (w0[1], w0[2], offset); + w0[2] = amd_bytealign_S (w0[0], w0[1], offset); + w0[1] = amd_bytealign_S ( 0, w0[0], offset); + w0[0] = 0; + break; + + case 2: + w3[2] = amd_bytealign_S (w2[3], 0, offset); + w3[1] = amd_bytealign_S (w2[2], w2[3], offset); + w3[0] = amd_bytealign_S (w2[1], w2[2], offset); + w2[3] = amd_bytealign_S (w2[0], w2[1], offset); + w2[2] = amd_bytealign_S (w1[3], w2[0], offset); + w2[1] = amd_bytealign_S (w1[2], w1[3], offset); + w2[0] = amd_bytealign_S (w1[1], w1[2], offset); + w1[3] = amd_bytealign_S (w1[0], w1[1], offset); + w1[2] = amd_bytealign_S (w0[3], w1[0], offset); + w1[1] = amd_bytealign_S (w0[2], w0[3], offset); + w1[0] = amd_bytealign_S (w0[1], w0[2], offset); + w0[3] = amd_bytealign_S (w0[0], w0[1], offset); + w0[2] = amd_bytealign_S ( 0, w0[0], offset); + w0[1] = 0; + w0[0] = 0; + break; + + case 3: + w3[2] = amd_bytealign_S (w2[2], 0, offset); + w3[1] = amd_bytealign_S (w2[1], w2[2], offset); + w3[0] = amd_bytealign_S (w2[0], w2[1], offset); + w2[3] = amd_bytealign_S (w1[3], w2[0], offset); + w2[2] = amd_bytealign_S (w1[2], w1[3], offset); + w2[1] = amd_bytealign_S (w1[1], w1[2], offset); + w2[0] = amd_bytealign_S (w1[0], w1[1], offset); + w1[3] = amd_bytealign_S (w0[3], w1[0], offset); + w1[2] = amd_bytealign_S (w0[2], w0[3], offset); + w1[1] = amd_bytealign_S (w0[1], w0[2], offset); + w1[0] = amd_bytealign_S (w0[0], w0[1], offset); + w0[3] = amd_bytealign_S ( 0, w0[0], offset); + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 4: + w3[2] = amd_bytealign_S (w2[1], 0, offset); + w3[1] = amd_bytealign_S (w2[0], w2[1], offset); + w3[0] = amd_bytealign_S (w1[3], w2[0], offset); + w2[3] = amd_bytealign_S (w1[2], w1[3], offset); + w2[2] = amd_bytealign_S (w1[1], w1[2], offset); + w2[1] = amd_bytealign_S (w1[0], w1[1], offset); + w2[0] = amd_bytealign_S (w0[3], w1[0], offset); + w1[3] = amd_bytealign_S (w0[2], w0[3], offset); + w1[2] = amd_bytealign_S (w0[1], w0[2], offset); + w1[1] = amd_bytealign_S (w0[0], w0[1], offset); + w1[0] = amd_bytealign_S ( 0, w0[0], offset); + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 5: + w3[2] = amd_bytealign_S (w2[0], 0, offset); + w3[1] = amd_bytealign_S (w1[3], w2[0], offset); + w3[0] = amd_bytealign_S (w1[2], w1[3], offset); + w2[3] = amd_bytealign_S (w1[1], w1[2], offset); + w2[2] = amd_bytealign_S (w1[0], w1[1], offset); + w2[1] = amd_bytealign_S (w0[3], w1[0], offset); + w2[0] = amd_bytealign_S (w0[2], w0[3], offset); + w1[3] = amd_bytealign_S (w0[1], w0[2], offset); + w1[2] = amd_bytealign_S (w0[0], w0[1], offset); + w1[1] = amd_bytealign_S ( 0, w0[0], offset); + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 6: + w3[2] = amd_bytealign_S (w1[3], 0, offset); + w3[1] = amd_bytealign_S (w1[2], w1[3], offset); + w3[0] = amd_bytealign_S (w1[1], w1[2], offset); + w2[3] = amd_bytealign_S (w1[0], w1[1], offset); + w2[2] = amd_bytealign_S (w0[3], w1[0], offset); + w2[1] = amd_bytealign_S (w0[2], w0[3], offset); + w2[0] = amd_bytealign_S (w0[1], w0[2], offset); + w1[3] = amd_bytealign_S (w0[0], w0[1], offset); + w1[2] = amd_bytealign_S ( 0, w0[0], offset); + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 7: + w3[2] = amd_bytealign_S (w1[2], 0, offset); + w3[1] = amd_bytealign_S (w1[1], w1[2], offset); + w3[0] = amd_bytealign_S (w1[0], w1[1], offset); + w2[3] = amd_bytealign_S (w0[3], w1[0], offset); + w2[2] = amd_bytealign_S (w0[2], w0[3], offset); + w2[1] = amd_bytealign_S (w0[1], w0[2], offset); + w2[0] = amd_bytealign_S (w0[0], w0[1], offset); + w1[3] = amd_bytealign_S ( 0, w0[0], offset); + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 8: + w3[2] = amd_bytealign_S (w1[1], 0, offset); + w3[1] = amd_bytealign_S (w1[0], w1[1], offset); + w3[0] = amd_bytealign_S (w0[3], w1[0], offset); + w2[3] = amd_bytealign_S (w0[2], w0[3], offset); + w2[2] = amd_bytealign_S (w0[1], w0[2], offset); + w2[1] = amd_bytealign_S (w0[0], w0[1], offset); + w2[0] = amd_bytealign_S ( 0, w0[0], offset); + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 9: + w3[2] = amd_bytealign_S (w1[0], 0, offset); + w3[1] = amd_bytealign_S (w0[3], w1[0], offset); + w3[0] = amd_bytealign_S (w0[2], w0[3], offset); + w2[3] = amd_bytealign_S (w0[1], w0[2], offset); + w2[2] = amd_bytealign_S (w0[0], w0[1], offset); + w2[1] = amd_bytealign_S ( 0, w0[0], offset); + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 10: + w3[2] = amd_bytealign_S (w0[3], 0, offset); + w3[1] = amd_bytealign_S (w0[2], w0[3], offset); + w3[0] = amd_bytealign_S (w0[1], w0[2], offset); + w2[3] = amd_bytealign_S (w0[0], w0[1], offset); + w2[2] = amd_bytealign_S ( 0, w0[0], offset); + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 11: + w3[2] = amd_bytealign_S (w0[2], 0, offset); + w3[1] = amd_bytealign_S (w0[1], w0[2], offset); + w3[0] = amd_bytealign_S (w0[0], w0[1], offset); + w2[3] = amd_bytealign_S ( 0, w0[0], offset); + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 12: + w3[2] = amd_bytealign_S (w0[1], 0, offset); + w3[1] = amd_bytealign_S (w0[0], w0[1], offset); + w3[0] = amd_bytealign_S ( 0, w0[0], offset); + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 13: + w3[2] = amd_bytealign_S (w0[0], 0, offset); + w3[1] = amd_bytealign_S ( 0, w0[0], offset); + w3[0] = 0; + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + } + #endif + + #ifdef IS_NV + const int selector = (0x76543210 >> ((offset & 3) * 4)) & 0xffff; + + switch (offset / 4) + { + case 0: + w3[1] = __byte_perm_S (w3[1], w3[0], selector); + w3[0] = __byte_perm_S (w3[0], w2[3], selector); + w2[3] = __byte_perm_S (w2[3], w2[2], selector); + w2[2] = __byte_perm_S (w2[2], w2[1], selector); + w2[1] = __byte_perm_S (w2[1], w2[0], selector); + w2[0] = __byte_perm_S (w2[0], w1[3], selector); + w1[3] = __byte_perm_S (w1[3], w1[2], selector); + w1[2] = __byte_perm_S (w1[2], w1[1], selector); + w1[1] = __byte_perm_S (w1[1], w1[0], selector); + w1[0] = __byte_perm_S (w1[0], w0[3], selector); + w0[3] = __byte_perm_S (w0[3], w0[2], selector); + w0[2] = __byte_perm_S (w0[2], w0[1], selector); + w0[1] = __byte_perm_S (w0[1], w0[0], selector); + w0[0] = __byte_perm_S (w0[0], 0, selector); + break; + + case 1: + w3[1] = __byte_perm_S (w3[0], w2[3], selector); + w3[0] = __byte_perm_S (w2[3], w2[2], selector); + w2[3] = __byte_perm_S (w2[2], w2[1], selector); + w2[2] = __byte_perm_S (w2[1], w2[0], selector); + w2[1] = __byte_perm_S (w2[0], w1[3], selector); + w2[0] = __byte_perm_S (w1[3], w1[2], selector); + w1[3] = __byte_perm_S (w1[2], w1[1], selector); + w1[2] = __byte_perm_S (w1[1], w1[0], selector); + w1[1] = __byte_perm_S (w1[0], w0[3], selector); + w1[0] = __byte_perm_S (w0[3], w0[2], selector); + w0[3] = __byte_perm_S (w0[2], w0[1], selector); + w0[2] = __byte_perm_S (w0[1], w0[0], selector); + w0[1] = __byte_perm_S (w0[0], 0, selector); + w0[0] = 0; + break; + + case 2: + w3[1] = __byte_perm_S (w2[3], w2[2], selector); + w3[0] = __byte_perm_S (w2[2], w2[1], selector); + w2[3] = __byte_perm_S (w2[1], w2[0], selector); + w2[2] = __byte_perm_S (w2[0], w1[3], selector); + w2[1] = __byte_perm_S (w1[3], w1[2], selector); + w2[0] = __byte_perm_S (w1[2], w1[1], selector); + w1[3] = __byte_perm_S (w1[1], w1[0], selector); + w1[2] = __byte_perm_S (w1[0], w0[3], selector); + w1[1] = __byte_perm_S (w0[3], w0[2], selector); + w1[0] = __byte_perm_S (w0[2], w0[1], selector); + w0[3] = __byte_perm_S (w0[1], w0[0], selector); + w0[2] = __byte_perm_S (w0[0], 0, selector); + w0[1] = 0; + w0[0] = 0; + break; + + case 3: + w3[1] = __byte_perm_S (w2[2], w2[1], selector); + w3[0] = __byte_perm_S (w2[1], w2[0], selector); + w2[3] = __byte_perm_S (w2[0], w1[3], selector); + w2[2] = __byte_perm_S (w1[3], w1[2], selector); + w2[1] = __byte_perm_S (w1[2], w1[1], selector); + w2[0] = __byte_perm_S (w1[1], w1[0], selector); + w1[3] = __byte_perm_S (w1[0], w0[3], selector); + w1[2] = __byte_perm_S (w0[3], w0[2], selector); + w1[1] = __byte_perm_S (w0[2], w0[1], selector); + w1[0] = __byte_perm_S (w0[1], w0[0], selector); + w0[3] = __byte_perm_S (w0[0], 0, selector); + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 4: + w3[1] = __byte_perm_S (w2[1], w2[0], selector); + w3[0] = __byte_perm_S (w2[0], w1[3], selector); + w2[3] = __byte_perm_S (w1[3], w1[2], selector); + w2[2] = __byte_perm_S (w1[2], w1[1], selector); + w2[1] = __byte_perm_S (w1[1], w1[0], selector); + w2[0] = __byte_perm_S (w1[0], w0[3], selector); + w1[3] = __byte_perm_S (w0[3], w0[2], selector); + w1[2] = __byte_perm_S (w0[2], w0[1], selector); + w1[1] = __byte_perm_S (w0[1], w0[0], selector); + w1[0] = __byte_perm_S (w0[0], 0, selector); + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 5: + w3[1] = __byte_perm_S (w2[0], w1[3], selector); + w3[0] = __byte_perm_S (w1[3], w1[2], selector); + w2[3] = __byte_perm_S (w1[2], w1[1], selector); + w2[2] = __byte_perm_S (w1[1], w1[0], selector); + w2[1] = __byte_perm_S (w1[0], w0[3], selector); + w2[0] = __byte_perm_S (w0[3], w0[2], selector); + w1[3] = __byte_perm_S (w0[2], w0[1], selector); + w1[2] = __byte_perm_S (w0[1], w0[0], selector); + w1[1] = __byte_perm_S (w0[0], 0, selector); + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 6: + w3[1] = __byte_perm_S (w1[3], w1[2], selector); + w3[0] = __byte_perm_S (w1[2], w1[1], selector); + w2[3] = __byte_perm_S (w1[1], w1[0], selector); + w2[2] = __byte_perm_S (w1[0], w0[3], selector); + w2[1] = __byte_perm_S (w0[3], w0[2], selector); + w2[0] = __byte_perm_S (w0[2], w0[1], selector); + w1[3] = __byte_perm_S (w0[1], w0[0], selector); + w1[2] = __byte_perm_S (w0[0], 0, selector); + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 7: + w3[1] = __byte_perm_S (w1[2], w1[1], selector); + w3[0] = __byte_perm_S (w1[1], w1[0], selector); + w2[3] = __byte_perm_S (w1[0], w0[3], selector); + w2[2] = __byte_perm_S (w0[3], w0[2], selector); + w2[1] = __byte_perm_S (w0[2], w0[1], selector); + w2[0] = __byte_perm_S (w0[1], w0[0], selector); + w1[3] = __byte_perm_S (w0[0], 0, selector); + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 8: + w3[1] = __byte_perm_S (w1[1], w1[0], selector); + w3[0] = __byte_perm_S (w1[0], w0[3], selector); + w2[3] = __byte_perm_S (w0[3], w0[2], selector); + w2[2] = __byte_perm_S (w0[2], w0[1], selector); + w2[1] = __byte_perm_S (w0[1], w0[0], selector); + w2[0] = __byte_perm_S (w0[0], 0, selector); + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 9: + w3[1] = __byte_perm_S (w1[0], w0[3], selector); + w3[0] = __byte_perm_S (w0[3], w0[2], selector); + w2[3] = __byte_perm_S (w0[2], w0[1], selector); + w2[2] = __byte_perm_S (w0[1], w0[0], selector); + w2[1] = __byte_perm_S (w0[0], 0, selector); + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 10: + w3[1] = __byte_perm_S (w0[3], w0[2], selector); + w3[0] = __byte_perm_S (w0[2], w0[1], selector); + w2[3] = __byte_perm_S (w0[1], w0[0], selector); + w2[2] = __byte_perm_S (w0[0], 0, selector); + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 11: + w3[1] = __byte_perm_S (w0[2], w0[1], selector); + w3[0] = __byte_perm_S (w0[1], w0[0], selector); + w2[3] = __byte_perm_S (w0[0], 0, selector); + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 12: + w3[1] = __byte_perm_S (w0[1], w0[0], selector); + w3[0] = __byte_perm_S (w0[0], 0, selector); + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + + case 13: + w3[1] = __byte_perm_S (w0[0], 0, selector); + w3[0] = 0; + w2[3] = 0; + w2[2] = 0; + w2[1] = 0; + w2[0] = 0; + w1[3] = 0; + w1[2] = 0; + w1[1] = 0; + w1[0] = 0; + w0[3] = 0; + w0[2] = 0; + w0[1] = 0; + w0[0] = 0; + break; + } + #endif +} + +/** + * vector functions on scalar types (for inner loop usage) + */ + +#define PACKVS2(sn,vn,e) \ + sn[0] = vn[0].s##e; \ + sn[1] = vn[1].s##e; + +#define PACKSV2(sn,vn,e) \ + vn[0].s##e = sn[0]; \ + vn[1].s##e = sn[1]; + +#define PACKVS24(s0,s1,v0,v1,e) \ + PACKVS4 (s0, v0, e); \ + PACKVS4 (s1, v1, e); + +#define PACKSV24(s0,s1,v0,v1,e) \ + PACKSV4 (s0, v0, e); \ + PACKSV4 (s1, v1, e); + +#define PACKVS4(sn,vn,e) \ + sn[0] = vn[0].s##e; \ + sn[1] = vn[1].s##e; \ + sn[2] = vn[2].s##e; \ + sn[3] = vn[3].s##e; + +#define PACKSV4(sn,vn,e) \ + vn[0].s##e = sn[0]; \ + vn[1].s##e = sn[1]; \ + vn[2].s##e = sn[2]; \ + vn[3].s##e = sn[3]; + +#define PACKVS44(s0,s1,s2,s3,v0,v1,v2,v3,e) \ + PACKVS4 (s0, v0, e); \ + PACKVS4 (s1, v1, e); \ + PACKVS4 (s2, v2, e); \ + PACKVS4 (s3, v3, e); + +#define PACKSV44(s0,s1,s2,s3,v0,v1,v2,v3,e) \ + PACKSV4 (s0, v0, e); \ + PACKSV4 (s1, v1, e); \ + PACKSV4 (s2, v2, e); \ + PACKSV4 (s3, v3, e); + +inline void switch_buffer_by_offset_le_VV (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x offset) +{ + #if VECT_SIZE == 1 + + switch_buffer_by_offset_le_S (w0, w1, w2, w3, offset); + + #else + + u32 t0[4]; + u32 t1[4]; + u32 t2[4]; + u32 t3[4]; + + #endif + + #if VECT_SIZE == 2 + + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); + + #elif VECT_SIZE == 4 + + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); + + #elif VECT_SIZE == 8 + + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); + + #elif VECT_SIZE == 16 + + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s8); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s9); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, a); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sa); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, a); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, b); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sb); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, b); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, c); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sc); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, c); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, d); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sd); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, d); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, e); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.se); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, e); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, f); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sf); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, f); + + #endif +} + +inline void append_0x01_2x4_VV (u32x w0[4], u32x w1[4], const u32x offset) +{ + #if VECT_SIZE == 1 + + append_0x01_2x4_S (w0, w1, offset); + + #else + + u32 t0[4]; + u32 t1[4]; + + #endif + + #if VECT_SIZE == 2 + + PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); + PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); + + #elif VECT_SIZE == 4 + + PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); + PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); + PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); + PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); + + #elif VECT_SIZE == 8 + + PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); + PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); + PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); + PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); + PACKVS24 (t0, t1, w0, w1, 4); append_0x01_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4); + PACKVS24 (t0, t1, w0, w1, 5); append_0x01_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5); + PACKVS24 (t0, t1, w0, w1, 6); append_0x01_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6); + PACKVS24 (t0, t1, w0, w1, 7); append_0x01_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7); + + #elif VECT_SIZE == 16 + + PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); + PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); + PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); + PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); + PACKVS24 (t0, t1, w0, w1, 4); append_0x01_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4); + PACKVS24 (t0, t1, w0, w1, 5); append_0x01_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5); + PACKVS24 (t0, t1, w0, w1, 6); append_0x01_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6); + PACKVS24 (t0, t1, w0, w1, 7); append_0x01_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7); + PACKVS24 (t0, t1, w0, w1, 8); append_0x01_2x4_S (t0, t1, offset.s8); PACKSV24 (t0, t1, w0, w1, 8); + PACKVS24 (t0, t1, w0, w1, 9); append_0x01_2x4_S (t0, t1, offset.s9); PACKSV24 (t0, t1, w0, w1, 9); + PACKVS24 (t0, t1, w0, w1, a); append_0x01_2x4_S (t0, t1, offset.sa); PACKSV24 (t0, t1, w0, w1, a); + PACKVS24 (t0, t1, w0, w1, b); append_0x01_2x4_S (t0, t1, offset.sb); PACKSV24 (t0, t1, w0, w1, b); + PACKVS24 (t0, t1, w0, w1, c); append_0x01_2x4_S (t0, t1, offset.sc); PACKSV24 (t0, t1, w0, w1, c); + PACKVS24 (t0, t1, w0, w1, d); append_0x01_2x4_S (t0, t1, offset.sd); PACKSV24 (t0, t1, w0, w1, d); + PACKVS24 (t0, t1, w0, w1, e); append_0x01_2x4_S (t0, t1, offset.se); PACKSV24 (t0, t1, w0, w1, e); + PACKVS24 (t0, t1, w0, w1, f); append_0x01_2x4_S (t0, t1, offset.sf); PACKSV24 (t0, t1, w0, w1, f); + + #endif +} + +inline void append_0x80_2x4_VV (u32x w0[4], u32x w1[4], const u32x offset) +{ + #if VECT_SIZE == 1 + + append_0x80_2x4_S (w0, w1, offset); + + #else + + u32 t0[4]; + u32 t1[4]; + + #endif + + #if VECT_SIZE == 2 + + PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); + PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); + + #elif VECT_SIZE == 4 + + PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); + PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); + PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); + PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); + + #elif VECT_SIZE == 8 + + PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); + PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); + PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); + PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); + PACKVS24 (t0, t1, w0, w1, 4); append_0x80_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4); + PACKVS24 (t0, t1, w0, w1, 5); append_0x80_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5); + PACKVS24 (t0, t1, w0, w1, 6); append_0x80_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6); + PACKVS24 (t0, t1, w0, w1, 7); append_0x80_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7); + + #elif VECT_SIZE == 16 + + PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0); + PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1); + PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2); + PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3); + PACKVS24 (t0, t1, w0, w1, 4); append_0x80_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4); + PACKVS24 (t0, t1, w0, w1, 5); append_0x80_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5); + PACKVS24 (t0, t1, w0, w1, 6); append_0x80_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6); + PACKVS24 (t0, t1, w0, w1, 7); append_0x80_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7); + PACKVS24 (t0, t1, w0, w1, 8); append_0x80_2x4_S (t0, t1, offset.s8); PACKSV24 (t0, t1, w0, w1, 8); + PACKVS24 (t0, t1, w0, w1, 9); append_0x80_2x4_S (t0, t1, offset.s9); PACKSV24 (t0, t1, w0, w1, 9); + PACKVS24 (t0, t1, w0, w1, a); append_0x80_2x4_S (t0, t1, offset.sa); PACKSV24 (t0, t1, w0, w1, a); + PACKVS24 (t0, t1, w0, w1, b); append_0x80_2x4_S (t0, t1, offset.sb); PACKSV24 (t0, t1, w0, w1, b); + PACKVS24 (t0, t1, w0, w1, c); append_0x80_2x4_S (t0, t1, offset.sc); PACKSV24 (t0, t1, w0, w1, c); + PACKVS24 (t0, t1, w0, w1, d); append_0x80_2x4_S (t0, t1, offset.sd); PACKSV24 (t0, t1, w0, w1, d); + PACKVS24 (t0, t1, w0, w1, e); append_0x80_2x4_S (t0, t1, offset.se); PACKSV24 (t0, t1, w0, w1, e); + PACKVS24 (t0, t1, w0, w1, f); append_0x80_2x4_S (t0, t1, offset.sf); PACKSV24 (t0, t1, w0, w1, f); + + #endif +} + +inline void append_0x80_4x4_VV (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x offset) +{ + #if VECT_SIZE == 1 + + append_0x80_4x4_S (w0, w1, w2, w3, offset); + + #else + + u32 t0[4]; + u32 t1[4]; + u32 t2[4]; + u32 t3[4]; + + #endif + + #if VECT_SIZE == 2 + + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); + + #elif VECT_SIZE == 4 + + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); + + #elif VECT_SIZE == 8 + + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); append_0x80_4x4_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); append_0x80_4x4_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); append_0x80_4x4_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); append_0x80_4x4_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); + + #elif VECT_SIZE == 16 + + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); append_0x80_4x4_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); append_0x80_4x4_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); append_0x80_4x4_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); append_0x80_4x4_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); append_0x80_4x4_S (t0, t1, t2, t3, offset.s8); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); append_0x80_4x4_S (t0, t1, t2, t3, offset.s9); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, a); append_0x80_4x4_S (t0, t1, t2, t3, offset.sa); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, a); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, b); append_0x80_4x4_S (t0, t1, t2, t3, offset.sb); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, b); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, c); append_0x80_4x4_S (t0, t1, t2, t3, offset.sc); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, c); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, d); append_0x80_4x4_S (t0, t1, t2, t3, offset.sd); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, d); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, e); append_0x80_4x4_S (t0, t1, t2, t3, offset.se); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, e); + PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, f); append_0x80_4x4_S (t0, t1, t2, t3, offset.sf); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, f); + + #endif +} diff --git a/OpenCL/inc_comp_multi.cl b/OpenCL/inc_comp_multi.cl new file mode 100644 index 0000000..14ad55c --- /dev/null +++ b/OpenCL/inc_comp_multi.cl @@ -0,0 +1,32 @@ +u32 digest_tp[4]; + +digest_tp[0] = r0; +digest_tp[1] = r1; +digest_tp[2] = r2; +digest_tp[3] = r3; + +if (check (digest_tp, + bitmaps_buf_s1_a, + bitmaps_buf_s1_b, + bitmaps_buf_s1_c, + bitmaps_buf_s1_d, + bitmaps_buf_s2_a, + bitmaps_buf_s2_b, + bitmaps_buf_s2_c, + bitmaps_buf_s2_d, + bitmap_mask, + bitmap_shift1, + bitmap_shift2)) +{ + int digest_pos = find_hash (digest_tp, digests_cnt, &digests_buf[digests_offset]); + + if (digest_pos != -1) + { + const u32 final_hash_pos = digests_offset + digest_pos; + + if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) + { + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos); + } + } +} diff --git a/OpenCL/inc_comp_multi_bs.cl b/OpenCL/inc_comp_multi_bs.cl new file mode 100644 index 0000000..6546a97 --- /dev/null +++ b/OpenCL/inc_comp_multi_bs.cl @@ -0,0 +1,32 @@ +u32 digest_tp[4]; + +digest_tp[0] = r0; +digest_tp[1] = r1; +digest_tp[2] = r2; +digest_tp[3] = r3; + +if (check (digest_tp, + bitmaps_buf_s1_a, + bitmaps_buf_s1_b, + bitmaps_buf_s1_c, + bitmaps_buf_s1_d, + bitmaps_buf_s2_a, + bitmaps_buf_s2_b, + bitmaps_buf_s2_c, + bitmaps_buf_s2_d, + bitmap_mask, + bitmap_shift1, + bitmap_shift2)) +{ + int digest_pos = find_hash (digest_tp, digests_cnt, &digests_buf[digests_offset]); + + if (digest_pos != -1) + { + const u32 final_hash_pos = digests_offset + digest_pos; + + if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) + { + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + slice); + } + } +} diff --git a/OpenCL/inc_comp_single.cl b/OpenCL/inc_comp_single.cl new file mode 100644 index 0000000..631a8bc --- /dev/null +++ b/OpenCL/inc_comp_single.cl @@ -0,0 +1,12 @@ +if ((r0 == search[0]) + && (r1 == search[1]) + && (r2 == search[2]) + && (r3 == search[3])) +{ + const u32 final_hash_pos = digests_offset + 0; + + if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) + { + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos); + } +} diff --git a/OpenCL/inc_comp_single_bs.cl b/OpenCL/inc_comp_single_bs.cl new file mode 100644 index 0000000..5c081c1 --- /dev/null +++ b/OpenCL/inc_comp_single_bs.cl @@ -0,0 +1,6 @@ +const u32 final_hash_pos = digests_offset + 0; + +if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) +{ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos); +} diff --git a/OpenCL/inc_hash_constants.h b/OpenCL/inc_hash_constants.h new file mode 100644 index 0000000..5351445 --- /dev/null +++ b/OpenCL/inc_hash_constants.h @@ -0,0 +1,758 @@ +/** + * Author......: Jens Steube + * License.....: MIT + */ + +#define COMBINATOR_MODE_BASE_LEFT 10001 +#define COMBINATOR_MODE_BASE_RIGHT 10002 + +#ifdef SHARED_H +#define _BCRYPT_ +#define _SHA1_ +#define _SHA256_ +#define _SHA384_ +#define _SHA512_ +#define _MD4_ +#define _MD5_ +#define _MD5H_ +#define _KECCAK_ +#define _RIPEMD160_ +#define _WHIRLPOOL_ +#define _SAPB_ +#define _CLOUDKEY_ +#define _OFFICE2013_ +#define _SIPHASH_ +#define _PDF17L8_ +#define _PBKDF2_MD5_ +#define _PBKDF2_SHA1_ +#define _PBKDF2_SHA256_ +#define _PBKDF2_SHA512_ +#define _RAR3_ +#define _ZIP2_ +#endif + +#ifdef _SIPHASH_ +/** + * SipHash Constants + */ + +#define SIPHASHM_0 0x736f6d6570736575 +#define SIPHASHM_1 0x646f72616e646f6d +#define SIPHASHM_2 0x6c7967656e657261 +#define SIPHASHM_3 0x7465646279746573 +#endif + +#if defined _BCRYPT_ || defined _PSAFE2_ +/** + * bcrypt Constants + */ + +#define BCRYPTM_0 0x4F727068u +#define BCRYPTM_1 0x65616E42u +#define BCRYPTM_2 0x65686F6Cu +#define BCRYPTM_3 0x64657253u +#define BCRYPTM_4 0x63727944u +#define BCRYPTM_5 0x6F756274u +#endif + +#if defined _SHA1_ || defined _SAPG_ || defined _OFFICE2007_ || defined _OFFICE2010_ || defined _OLDOFFICE34_ || defined _ANDROIDFDE_ || defined _DCC2_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _PSAFE2_ || defined _LOTUS8_ || defined _PBKDF2_SHA1_ || defined _RAR3_ || defined _SHA256_SHA1_ || defined _ZIP2_ +/** + * SHA1 Constants + */ + +#define SHA1M_A 0x67452301u +#define SHA1M_B 0xefcdab89u +#define SHA1M_C 0x98badcfeu +#define SHA1M_D 0x10325476u +#define SHA1M_E 0xc3d2e1f0u + +#define SHA1C00 0x5a827999u +#define SHA1C01 0x6ed9eba1u +#define SHA1C02 0x8f1bbcdcu +#define SHA1C03 0xca62c1d6u +#endif + +#if defined _SHA256_ || defined _PDF17L8_ || defined _SEVEN_ZIP_ || defined _ANDROIDFDE_ || defined _CLOUDKEY_ || defined _SCRYPT_ || defined _PBKDF2_SHA256_ || defined _SHA256_SHA1_ || defined _MS_DRSR_ || defined _ANDROIDFDE_SAMSUNG_ || defined _RAR5_ || defined _KEEPASS_ +/** + * SHA256 Constants + */ + +#define SHA256M_A 0x6a09e667u +#define SHA256M_B 0xbb67ae85u +#define SHA256M_C 0x3c6ef372u +#define SHA256M_D 0xa54ff53au +#define SHA256M_E 0x510e527fu +#define SHA256M_F 0x9b05688cu +#define SHA256M_G 0x1f83d9abu +#define SHA256M_H 0x5be0cd19u + +#define SHA256C00 0x428a2f98u +#define SHA256C01 0x71374491u +#define SHA256C02 0xb5c0fbcfu +#define SHA256C03 0xe9b5dba5u +#define SHA256C04 0x3956c25bu +#define SHA256C05 0x59f111f1u +#define SHA256C06 0x923f82a4u +#define SHA256C07 0xab1c5ed5u +#define SHA256C08 0xd807aa98u +#define SHA256C09 0x12835b01u +#define SHA256C0a 0x243185beu +#define SHA256C0b 0x550c7dc3u +#define SHA256C0c 0x72be5d74u +#define SHA256C0d 0x80deb1feu +#define SHA256C0e 0x9bdc06a7u +#define SHA256C0f 0xc19bf174u +#define SHA256C10 0xe49b69c1u +#define SHA256C11 0xefbe4786u +#define SHA256C12 0x0fc19dc6u +#define SHA256C13 0x240ca1ccu +#define SHA256C14 0x2de92c6fu +#define SHA256C15 0x4a7484aau +#define SHA256C16 0x5cb0a9dcu +#define SHA256C17 0x76f988dau +#define SHA256C18 0x983e5152u +#define SHA256C19 0xa831c66du +#define SHA256C1a 0xb00327c8u +#define SHA256C1b 0xbf597fc7u +#define SHA256C1c 0xc6e00bf3u +#define SHA256C1d 0xd5a79147u +#define SHA256C1e 0x06ca6351u +#define SHA256C1f 0x14292967u +#define SHA256C20 0x27b70a85u +#define SHA256C21 0x2e1b2138u +#define SHA256C22 0x4d2c6dfcu +#define SHA256C23 0x53380d13u +#define SHA256C24 0x650a7354u +#define SHA256C25 0x766a0abbu +#define SHA256C26 0x81c2c92eu +#define SHA256C27 0x92722c85u +#define SHA256C28 0xa2bfe8a1u +#define SHA256C29 0xa81a664bu +#define SHA256C2a 0xc24b8b70u +#define SHA256C2b 0xc76c51a3u +#define SHA256C2c 0xd192e819u +#define SHA256C2d 0xd6990624u +#define SHA256C2e 0xf40e3585u +#define SHA256C2f 0x106aa070u +#define SHA256C30 0x19a4c116u +#define SHA256C31 0x1e376c08u +#define SHA256C32 0x2748774cu +#define SHA256C33 0x34b0bcb5u +#define SHA256C34 0x391c0cb3u +#define SHA256C35 0x4ed8aa4au +#define SHA256C36 0x5b9cca4fu +#define SHA256C37 0x682e6ff3u +#define SHA256C38 0x748f82eeu +#define SHA256C39 0x78a5636fu +#define SHA256C3a 0x84c87814u +#define SHA256C3b 0x8cc70208u +#define SHA256C3c 0x90befffau +#define SHA256C3d 0xa4506cebu +#define SHA256C3e 0xbef9a3f7u +#define SHA256C3f 0xc67178f2u +#endif + +#if defined _MD4_ || defined _DCC2_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _MS_DRSR_ || defined _KRB5TGS_ +/** + * MD4 Constants + */ + +#define MD4M_A 0x67452301u +#define MD4M_B 0xefcdab89u +#define MD4M_C 0x98badcfeu +#define MD4M_D 0x10325476u + +#define MD4S00 3u +#define MD4S01 7u +#define MD4S02 11u +#define MD4S03 19u +#define MD4S10 3u +#define MD4S11 5u +#define MD4S12 9u +#define MD4S13 13u +#define MD4S20 3u +#define MD4S21 9u +#define MD4S22 11u +#define MD4S23 15u + +#define MD4C00 0x00000000u +#define MD4C01 0x5a827999u +#define MD4C02 0x6ed9eba1u +#endif + +#if defined _MD5_ || defined _MD5H_ || defined _SAPB_ || defined _OLDOFFICE01_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _PBKDF2_MD5_ || defined _KRB5TGS_ +/** + * MD5 Constants + */ + +#define MD5M_A 0x67452301u +#define MD5M_B 0xefcdab89u +#define MD5M_C 0x98badcfeu +#define MD5M_D 0x10325476u + +#define MD5S00 7u +#define MD5S01 12u +#define MD5S02 17u +#define MD5S03 22u +#define MD5S10 5u +#define MD5S11 9u +#define MD5S12 14u +#define MD5S13 20u +#define MD5S20 4u +#define MD5S21 11u +#define MD5S22 16u +#define MD5S23 23u +#define MD5S30 6u +#define MD5S31 10u +#define MD5S32 15u +#define MD5S33 21u + +#define MD5C00 0xd76aa478u +#define MD5C01 0xe8c7b756u +#define MD5C02 0x242070dbu +#define MD5C03 0xc1bdceeeu +#define MD5C04 0xf57c0fafu +#define MD5C05 0x4787c62au +#define MD5C06 0xa8304613u +#define MD5C07 0xfd469501u +#define MD5C08 0x698098d8u +#define MD5C09 0x8b44f7afu +#define MD5C0a 0xffff5bb1u +#define MD5C0b 0x895cd7beu +#define MD5C0c 0x6b901122u +#define MD5C0d 0xfd987193u +#define MD5C0e 0xa679438eu +#define MD5C0f 0x49b40821u +#define MD5C10 0xf61e2562u +#define MD5C11 0xc040b340u +#define MD5C12 0x265e5a51u +#define MD5C13 0xe9b6c7aau +#define MD5C14 0xd62f105du +#define MD5C15 0x02441453u +#define MD5C16 0xd8a1e681u +#define MD5C17 0xe7d3fbc8u +#define MD5C18 0x21e1cde6u +#define MD5C19 0xc33707d6u +#define MD5C1a 0xf4d50d87u +#define MD5C1b 0x455a14edu +#define MD5C1c 0xa9e3e905u +#define MD5C1d 0xfcefa3f8u +#define MD5C1e 0x676f02d9u +#define MD5C1f 0x8d2a4c8au +#define MD5C20 0xfffa3942u +#define MD5C21 0x8771f681u +#define MD5C22 0x6d9d6122u +#define MD5C23 0xfde5380cu +#define MD5C24 0xa4beea44u +#define MD5C25 0x4bdecfa9u +#define MD5C26 0xf6bb4b60u +#define MD5C27 0xbebfbc70u +#define MD5C28 0x289b7ec6u +#define MD5C29 0xeaa127fau +#define MD5C2a 0xd4ef3085u +#define MD5C2b 0x04881d05u +#define MD5C2c 0xd9d4d039u +#define MD5C2d 0xe6db99e5u +#define MD5C2e 0x1fa27cf8u +#define MD5C2f 0xc4ac5665u +#define MD5C30 0xf4292244u +#define MD5C31 0x432aff97u +#define MD5C32 0xab9423a7u +#define MD5C33 0xfc93a039u +#define MD5C34 0x655b59c3u +#define MD5C35 0x8f0ccc92u +#define MD5C36 0xffeff47du +#define MD5C37 0x85845dd1u +#define MD5C38 0x6fa87e4fu +#define MD5C39 0xfe2ce6e0u +#define MD5C3a 0xa3014314u +#define MD5C3b 0x4e0811a1u +#define MD5C3c 0xf7537e82u +#define MD5C3d 0xbd3af235u +#define MD5C3e 0x2ad7d2bbu +#define MD5C3f 0xeb86d391u +#endif + +#if defined _SHA384_ || defined _PDF17L8_ +/** + * SHA384 Constants (64 bits) + */ + +#define SHA384M_A 0xcbbb9d5dc1059ed8 +#define SHA384M_B 0x629a292a367cd507 +#define SHA384M_C 0x9159015a3070dd17 +#define SHA384M_D 0x152fecd8f70e5939 +#define SHA384M_E 0x67332667ffc00b31 +#define SHA384M_F 0x8eb44a8768581511 +#define SHA384M_G 0xdb0c2e0d64f98fa7 +#define SHA384M_H 0x47b5481dbefa4fa4 + +#define SHA384C00 0x428a2f98d728ae22 +#define SHA384C01 0x7137449123ef65cd +#define SHA384C02 0xb5c0fbcfec4d3b2f +#define SHA384C03 0xe9b5dba58189dbbc +#define SHA384C04 0x3956c25bf348b538 +#define SHA384C05 0x59f111f1b605d019 +#define SHA384C06 0x923f82a4af194f9b +#define SHA384C07 0xab1c5ed5da6d8118 +#define SHA384C08 0xd807aa98a3030242 +#define SHA384C09 0x12835b0145706fbe +#define SHA384C0a 0x243185be4ee4b28c +#define SHA384C0b 0x550c7dc3d5ffb4e2 +#define SHA384C0c 0x72be5d74f27b896f +#define SHA384C0d 0x80deb1fe3b1696b1 +#define SHA384C0e 0x9bdc06a725c71235 +#define SHA384C0f 0xc19bf174cf692694 +#define SHA384C10 0xe49b69c19ef14ad2 +#define SHA384C11 0xefbe4786384f25e3 +#define SHA384C12 0x0fc19dc68b8cd5b5 +#define SHA384C13 0x240ca1cc77ac9c65 +#define SHA384C14 0x2de92c6f592b0275 +#define SHA384C15 0x4a7484aa6ea6e483 +#define SHA384C16 0x5cb0a9dcbd41fbd4 +#define SHA384C17 0x76f988da831153b5 +#define SHA384C18 0x983e5152ee66dfab +#define SHA384C19 0xa831c66d2db43210 +#define SHA384C1a 0xb00327c898fb213f +#define SHA384C1b 0xbf597fc7beef0ee4 +#define SHA384C1c 0xc6e00bf33da88fc2 +#define SHA384C1d 0xd5a79147930aa725 +#define SHA384C1e 0x06ca6351e003826f +#define SHA384C1f 0x142929670a0e6e70 +#define SHA384C20 0x27b70a8546d22ffc +#define SHA384C21 0x2e1b21385c26c926 +#define SHA384C22 0x4d2c6dfc5ac42aed +#define SHA384C23 0x53380d139d95b3df +#define SHA384C24 0x650a73548baf63de +#define SHA384C25 0x766a0abb3c77b2a8 +#define SHA384C26 0x81c2c92e47edaee6 +#define SHA384C27 0x92722c851482353b +#define SHA384C28 0xa2bfe8a14cf10364 +#define SHA384C29 0xa81a664bbc423001 +#define SHA384C2a 0xc24b8b70d0f89791 +#define SHA384C2b 0xc76c51a30654be30 +#define SHA384C2c 0xd192e819d6ef5218 +#define SHA384C2d 0xd69906245565a910 +#define SHA384C2e 0xf40e35855771202a +#define SHA384C2f 0x106aa07032bbd1b8 +#define SHA384C30 0x19a4c116b8d2d0c8 +#define SHA384C31 0x1e376c085141ab53 +#define SHA384C32 0x2748774cdf8eeb99 +#define SHA384C33 0x34b0bcb5e19b48a8 +#define SHA384C34 0x391c0cb3c5c95a63 +#define SHA384C35 0x4ed8aa4ae3418acb +#define SHA384C36 0x5b9cca4f7763e373 +#define SHA384C37 0x682e6ff3d6b2b8a3 +#define SHA384C38 0x748f82ee5defb2fc +#define SHA384C39 0x78a5636f43172f60 +#define SHA384C3a 0x84c87814a1f0ab72 +#define SHA384C3b 0x8cc702081a6439ec +#define SHA384C3c 0x90befffa23631e28 +#define SHA384C3d 0xa4506cebde82bde9 +#define SHA384C3e 0xbef9a3f7b2c67915 +#define SHA384C3f 0xc67178f2e372532b +#define SHA384C40 0xca273eceea26619c +#define SHA384C41 0xd186b8c721c0c207 +#define SHA384C42 0xeada7dd6cde0eb1e +#define SHA384C43 0xf57d4f7fee6ed178 +#define SHA384C44 0x06f067aa72176fba +#define SHA384C45 0x0a637dc5a2c898a6 +#define SHA384C46 0x113f9804bef90dae +#define SHA384C47 0x1b710b35131c471b +#define SHA384C48 0x28db77f523047d84 +#define SHA384C49 0x32caab7b40c72493 +#define SHA384C4a 0x3c9ebe0a15c9bebc +#define SHA384C4b 0x431d67c49c100d4c +#define SHA384C4c 0x4cc5d4becb3e42b6 +#define SHA384C4d 0x597f299cfc657e2a +#define SHA384C4e 0x5fcb6fab3ad6faec +#define SHA384C4f 0x6c44198c4a475817 + +#endif + +#if defined _SHA512_ || defined _CLOUDKEY_ || defined _OFFICE2013_ || defined _PDF17L8_ || defined _PBKDF2_SHA512_ +/** + * SHA512 Constants (64 bits) + */ + +#define SHA512M_A 0x6a09e667f3bcc908 +#define SHA512M_B 0xbb67ae8584caa73b +#define SHA512M_C 0x3c6ef372fe94f82b +#define SHA512M_D 0xa54ff53a5f1d36f1 +#define SHA512M_E 0x510e527fade682d1 +#define SHA512M_F 0x9b05688c2b3e6c1f +#define SHA512M_G 0x1f83d9abfb41bd6b +#define SHA512M_H 0x5be0cd19137e2179 + +#define SHA512C00 0x428a2f98d728ae22 +#define SHA512C01 0x7137449123ef65cd +#define SHA512C02 0xb5c0fbcfec4d3b2f +#define SHA512C03 0xe9b5dba58189dbbc +#define SHA512C04 0x3956c25bf348b538 +#define SHA512C05 0x59f111f1b605d019 +#define SHA512C06 0x923f82a4af194f9b +#define SHA512C07 0xab1c5ed5da6d8118 +#define SHA512C08 0xd807aa98a3030242 +#define SHA512C09 0x12835b0145706fbe +#define SHA512C0a 0x243185be4ee4b28c +#define SHA512C0b 0x550c7dc3d5ffb4e2 +#define SHA512C0c 0x72be5d74f27b896f +#define SHA512C0d 0x80deb1fe3b1696b1 +#define SHA512C0e 0x9bdc06a725c71235 +#define SHA512C0f 0xc19bf174cf692694 +#define SHA512C10 0xe49b69c19ef14ad2 +#define SHA512C11 0xefbe4786384f25e3 +#define SHA512C12 0x0fc19dc68b8cd5b5 +#define SHA512C13 0x240ca1cc77ac9c65 +#define SHA512C14 0x2de92c6f592b0275 +#define SHA512C15 0x4a7484aa6ea6e483 +#define SHA512C16 0x5cb0a9dcbd41fbd4 +#define SHA512C17 0x76f988da831153b5 +#define SHA512C18 0x983e5152ee66dfab +#define SHA512C19 0xa831c66d2db43210 +#define SHA512C1a 0xb00327c898fb213f +#define SHA512C1b 0xbf597fc7beef0ee4 +#define SHA512C1c 0xc6e00bf33da88fc2 +#define SHA512C1d 0xd5a79147930aa725 +#define SHA512C1e 0x06ca6351e003826f +#define SHA512C1f 0x142929670a0e6e70 +#define SHA512C20 0x27b70a8546d22ffc +#define SHA512C21 0x2e1b21385c26c926 +#define SHA512C22 0x4d2c6dfc5ac42aed +#define SHA512C23 0x53380d139d95b3df +#define SHA512C24 0x650a73548baf63de +#define SHA512C25 0x766a0abb3c77b2a8 +#define SHA512C26 0x81c2c92e47edaee6 +#define SHA512C27 0x92722c851482353b +#define SHA512C28 0xa2bfe8a14cf10364 +#define SHA512C29 0xa81a664bbc423001 +#define SHA512C2a 0xc24b8b70d0f89791 +#define SHA512C2b 0xc76c51a30654be30 +#define SHA512C2c 0xd192e819d6ef5218 +#define SHA512C2d 0xd69906245565a910 +#define SHA512C2e 0xf40e35855771202a +#define SHA512C2f 0x106aa07032bbd1b8 +#define SHA512C30 0x19a4c116b8d2d0c8 +#define SHA512C31 0x1e376c085141ab53 +#define SHA512C32 0x2748774cdf8eeb99 +#define SHA512C33 0x34b0bcb5e19b48a8 +#define SHA512C34 0x391c0cb3c5c95a63 +#define SHA512C35 0x4ed8aa4ae3418acb +#define SHA512C36 0x5b9cca4f7763e373 +#define SHA512C37 0x682e6ff3d6b2b8a3 +#define SHA512C38 0x748f82ee5defb2fc +#define SHA512C39 0x78a5636f43172f60 +#define SHA512C3a 0x84c87814a1f0ab72 +#define SHA512C3b 0x8cc702081a6439ec +#define SHA512C3c 0x90befffa23631e28 +#define SHA512C3d 0xa4506cebde82bde9 +#define SHA512C3e 0xbef9a3f7b2c67915 +#define SHA512C3f 0xc67178f2e372532b +#define SHA512C40 0xca273eceea26619c +#define SHA512C41 0xd186b8c721c0c207 +#define SHA512C42 0xeada7dd6cde0eb1e +#define SHA512C43 0xf57d4f7fee6ed178 +#define SHA512C44 0x06f067aa72176fba +#define SHA512C45 0x0a637dc5a2c898a6 +#define SHA512C46 0x113f9804bef90dae +#define SHA512C47 0x1b710b35131c471b +#define SHA512C48 0x28db77f523047d84 +#define SHA512C49 0x32caab7b40c72493 +#define SHA512C4a 0x3c9ebe0a15c9bebc +#define SHA512C4b 0x431d67c49c100d4c +#define SHA512C4c 0x4cc5d4becb3e42b6 +#define SHA512C4d 0x597f299cfc657e2a +#define SHA512C4e 0x5fcb6fab3ad6faec +#define SHA512C4f 0x6c44198c4a475817 + +#define SHA512REV0 0x5218a97a1b97e8a0 +#define SHA512REV1 0x4334c1bea164f555 + +#endif + +#ifdef _RIPEMD160_ +/** + * RIPEMD160 Constants + */ + +#define RIPEMD160M_A 0x67452301u +#define RIPEMD160M_B 0xefcdab89u +#define RIPEMD160M_C 0x98badcfeu +#define RIPEMD160M_D 0x10325476u +#define RIPEMD160M_E 0xc3d2e1f0u + +#define RIPEMD160C00 0x00000000u +#define RIPEMD160C10 0x5a827999u +#define RIPEMD160C20 0x6ed9eba1u +#define RIPEMD160C30 0x8f1bbcdcu +#define RIPEMD160C40 0xa953fd4eu +#define RIPEMD160C50 0x50a28be6u +#define RIPEMD160C60 0x5c4dd124u +#define RIPEMD160C70 0x6d703ef3u +#define RIPEMD160C80 0x7a6d76e9u +#define RIPEMD160C90 0x00000000u + +#define RIPEMD160S00 11u +#define RIPEMD160S01 14u +#define RIPEMD160S02 15u +#define RIPEMD160S03 12u +#define RIPEMD160S04 5u +#define RIPEMD160S05 8u +#define RIPEMD160S06 7u +#define RIPEMD160S07 9u +#define RIPEMD160S08 11u +#define RIPEMD160S09 13u +#define RIPEMD160S0A 14u +#define RIPEMD160S0B 15u +#define RIPEMD160S0C 6u +#define RIPEMD160S0D 7u +#define RIPEMD160S0E 9u +#define RIPEMD160S0F 8u + +#define RIPEMD160S10 7u +#define RIPEMD160S11 6u +#define RIPEMD160S12 8u +#define RIPEMD160S13 13u +#define RIPEMD160S14 11u +#define RIPEMD160S15 9u +#define RIPEMD160S16 7u +#define RIPEMD160S17 15u +#define RIPEMD160S18 7u +#define RIPEMD160S19 12u +#define RIPEMD160S1A 15u +#define RIPEMD160S1B 9u +#define RIPEMD160S1C 11u +#define RIPEMD160S1D 7u +#define RIPEMD160S1E 13u +#define RIPEMD160S1F 12u + +#define RIPEMD160S20 11u +#define RIPEMD160S21 13u +#define RIPEMD160S22 6u +#define RIPEMD160S23 7u +#define RIPEMD160S24 14u +#define RIPEMD160S25 9u +#define RIPEMD160S26 13u +#define RIPEMD160S27 15u +#define RIPEMD160S28 14u +#define RIPEMD160S29 8u +#define RIPEMD160S2A 13u +#define RIPEMD160S2B 6u +#define RIPEMD160S2C 5u +#define RIPEMD160S2D 12u +#define RIPEMD160S2E 7u +#define RIPEMD160S2F 5u + +#define RIPEMD160S30 11u +#define RIPEMD160S31 12u +#define RIPEMD160S32 14u +#define RIPEMD160S33 15u +#define RIPEMD160S34 14u +#define RIPEMD160S35 15u +#define RIPEMD160S36 9u +#define RIPEMD160S37 8u +#define RIPEMD160S38 9u +#define RIPEMD160S39 14u +#define RIPEMD160S3A 5u +#define RIPEMD160S3B 6u +#define RIPEMD160S3C 8u +#define RIPEMD160S3D 6u +#define RIPEMD160S3E 5u +#define RIPEMD160S3F 12u + +#define RIPEMD160S40 9u +#define RIPEMD160S41 15u +#define RIPEMD160S42 5u +#define RIPEMD160S43 11u +#define RIPEMD160S44 6u +#define RIPEMD160S45 8u +#define RIPEMD160S46 13u +#define RIPEMD160S47 12u +#define RIPEMD160S48 5u +#define RIPEMD160S49 12u +#define RIPEMD160S4A 13u +#define RIPEMD160S4B 14u +#define RIPEMD160S4C 11u +#define RIPEMD160S4D 8u +#define RIPEMD160S4E 5u +#define RIPEMD160S4F 6u + +#define RIPEMD160S50 8u +#define RIPEMD160S51 9u +#define RIPEMD160S52 9u +#define RIPEMD160S53 11u +#define RIPEMD160S54 13u +#define RIPEMD160S55 15u +#define RIPEMD160S56 15u +#define RIPEMD160S57 5u +#define RIPEMD160S58 7u +#define RIPEMD160S59 7u +#define RIPEMD160S5A 8u +#define RIPEMD160S5B 11u +#define RIPEMD160S5C 14u +#define RIPEMD160S5D 14u +#define RIPEMD160S5E 12u +#define RIPEMD160S5F 6u + +#define RIPEMD160S60 9u +#define RIPEMD160S61 13u +#define RIPEMD160S62 15u +#define RIPEMD160S63 7u +#define RIPEMD160S64 12u +#define RIPEMD160S65 8u +#define RIPEMD160S66 9u +#define RIPEMD160S67 11u +#define RIPEMD160S68 7u +#define RIPEMD160S69 7u +#define RIPEMD160S6A 12u +#define RIPEMD160S6B 7u +#define RIPEMD160S6C 6u +#define RIPEMD160S6D 15u +#define RIPEMD160S6E 13u +#define RIPEMD160S6F 11u + +#define RIPEMD160S70 9u +#define RIPEMD160S71 7u +#define RIPEMD160S72 15u +#define RIPEMD160S73 11u +#define RIPEMD160S74 8u +#define RIPEMD160S75 6u +#define RIPEMD160S76 6u +#define RIPEMD160S77 14u +#define RIPEMD160S78 12u +#define RIPEMD160S79 13u +#define RIPEMD160S7A 5u +#define RIPEMD160S7B 14u +#define RIPEMD160S7C 13u +#define RIPEMD160S7D 13u +#define RIPEMD160S7E 7u +#define RIPEMD160S7F 5u + +#define RIPEMD160S80 15u +#define RIPEMD160S81 5u +#define RIPEMD160S82 8u +#define RIPEMD160S83 11u +#define RIPEMD160S84 14u +#define RIPEMD160S85 14u +#define RIPEMD160S86 6u +#define RIPEMD160S87 14u +#define RIPEMD160S88 6u +#define RIPEMD160S89 9u +#define RIPEMD160S8A 12u +#define RIPEMD160S8B 9u +#define RIPEMD160S8C 12u +#define RIPEMD160S8D 5u +#define RIPEMD160S8E 15u +#define RIPEMD160S8F 8u + +#define RIPEMD160S90 8u +#define RIPEMD160S91 5u +#define RIPEMD160S92 12u +#define RIPEMD160S93 9u +#define RIPEMD160S94 12u +#define RIPEMD160S95 5u +#define RIPEMD160S96 14u +#define RIPEMD160S97 6u +#define RIPEMD160S98 8u +#define RIPEMD160S99 13u +#define RIPEMD160S9A 6u +#define RIPEMD160S9B 5u +#define RIPEMD160S9C 15u +#define RIPEMD160S9D 13u +#define RIPEMD160S9E 11u +#define RIPEMD160S9F 11u + +#endif + +#ifdef _KECCAK_ +/** + * KECCAK Constants + */ + +#define KECCAK_RNDC_00 0x0000000000000001 +#define KECCAK_RNDC_01 0x0000000000008082 +#define KECCAK_RNDC_02 0x000000000000808a +#define KECCAK_RNDC_03 0x0000000080008000 +#define KECCAK_RNDC_04 0x000000000000808b +#define KECCAK_RNDC_05 0x0000000080000001 +#define KECCAK_RNDC_06 0x0000000080008081 +#define KECCAK_RNDC_07 0x0000000000008009 +#define KECCAK_RNDC_08 0x000000000000008a +#define KECCAK_RNDC_09 0x0000000000000088 +#define KECCAK_RNDC_10 0x0000000080008009 +#define KECCAK_RNDC_11 0x000000008000000a +#define KECCAK_RNDC_12 0x000000008000808b +#define KECCAK_RNDC_13 0x000000000000008b +#define KECCAK_RNDC_14 0x0000000000008089 +#define KECCAK_RNDC_15 0x0000000000008003 +#define KECCAK_RNDC_16 0x0000000000008002 +#define KECCAK_RNDC_17 0x0000000000000080 +#define KECCAK_RNDC_18 0x000000000000800a +#define KECCAK_RNDC_19 0x000000008000000a +#define KECCAK_RNDC_20 0x0000000080008081 +#define KECCAK_RNDC_21 0x0000000000008080 +#define KECCAK_RNDC_22 0x0000000080000001 +#define KECCAK_RNDC_23 0x0000000080008008 + +#define KECCAK_PILN_00 10 +#define KECCAK_PILN_01 7 +#define KECCAK_PILN_02 11 +#define KECCAK_PILN_03 17 +#define KECCAK_PILN_04 18 +#define KECCAK_PILN_05 3 +#define KECCAK_PILN_06 5 +#define KECCAK_PILN_07 16 +#define KECCAK_PILN_08 8 +#define KECCAK_PILN_09 21 +#define KECCAK_PILN_10 24 +#define KECCAK_PILN_11 4 +#define KECCAK_PILN_12 15 +#define KECCAK_PILN_13 23 +#define KECCAK_PILN_14 19 +#define KECCAK_PILN_15 13 +#define KECCAK_PILN_16 12 +#define KECCAK_PILN_17 2 +#define KECCAK_PILN_18 20 +#define KECCAK_PILN_19 14 +#define KECCAK_PILN_20 22 +#define KECCAK_PILN_21 9 +#define KECCAK_PILN_22 6 +#define KECCAK_PILN_23 1 + +#define KECCAK_ROTC_00 1 +#define KECCAK_ROTC_01 3 +#define KECCAK_ROTC_02 6 +#define KECCAK_ROTC_03 10 +#define KECCAK_ROTC_04 15 +#define KECCAK_ROTC_05 21 +#define KECCAK_ROTC_06 28 +#define KECCAK_ROTC_07 36 +#define KECCAK_ROTC_08 45 +#define KECCAK_ROTC_09 55 +#define KECCAK_ROTC_10 2 +#define KECCAK_ROTC_11 14 +#define KECCAK_ROTC_12 27 +#define KECCAK_ROTC_13 41 +#define KECCAK_ROTC_14 56 +#define KECCAK_ROTC_15 8 +#define KECCAK_ROTC_16 25 +#define KECCAK_ROTC_17 43 +#define KECCAK_ROTC_18 62 +#define KECCAK_ROTC_19 18 +#define KECCAK_ROTC_20 39 +#define KECCAK_ROTC_21 61 +#define KECCAK_ROTC_22 20 +#define KECCAK_ROTC_23 44 + + +#endif + +#ifdef _MYSQL323_ +/** + * MYSQL323 Constants + */ + +#define MYSQL323_A 0x50305735 +#define MYSQL323_B 0x12345671 + +#endif diff --git a/OpenCL/inc_hash_functions.cl b/OpenCL/inc_hash_functions.cl new file mode 100644 index 0000000..0ccbb25 --- /dev/null +++ b/OpenCL/inc_hash_functions.cl @@ -0,0 +1,413 @@ +/** + * Author......: Jens Steube + * License.....: MIT + */ + +#if defined _MD4_ || defined _DCC2_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _MS_DRSR_ || defined _KRB5TGS_ + +#define MD4_F_S(x,y,z) (((x) & (y)) | ((~(x)) & (z))) +#define MD4_G_S(x,y,z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) +#define MD4_H_S(x,y,z) ((x) ^ (y) ^ (z)) + +#ifdef IS_NV +#define MD4_F(x,y,z) (((x) & (y)) | ((~(x)) & (z))) +#define MD4_G(x,y,z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) +#define MD4_H(x,y,z) ((x) ^ (y) ^ (z)) +#define MD4_Fo(x,y,z) (MD4_F((x), (y), (z))) +#define MD4_Go(x,y,z) (MD4_G((x), (y), (z))) +#endif + +#ifdef IS_AMD +#define MD4_F(x,y,z) (((x) & (y)) | ((~(x)) & (z))) +#define MD4_G(x,y,z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) +#define MD4_H(x,y,z) ((x) ^ (y) ^ (z)) +#define MD4_Fo(x,y,z) (bitselect ((z), (y), (x))) +#define MD4_Go(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) +#endif + +#ifdef IS_GENERIC +#define MD4_F(x,y,z) (((x) & (y)) | ((~(x)) & (z))) +#define MD4_G(x,y,z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) +#define MD4_H(x,y,z) ((x) ^ (y) ^ (z)) +#define MD4_Fo(x,y,z) (MD4_F((x), (y), (z))) +#define MD4_Go(x,y,z) (MD4_G((x), (y), (z))) +#endif + +#define MD4_STEP_S(f,a,b,c,d,x,K,s) \ +{ \ + a += K; \ + a += x; \ + a += f (b, c, d); \ + a = rotl32_S (a, s); \ +} + +#define MD4_STEP(f,a,b,c,d,x,K,s) \ +{ \ + a += K; \ + a += x; \ + a += f (b, c, d); \ + a = rotl32 (a, s); \ +} + +#define MD4_STEP0(f,a,b,c,d,K,s) \ +{ \ + a += K; \ + a += f (b, c, d); \ + a = rotl32 (a, s); \ +} + +#endif + +#if defined _MD5_ || defined _MD5H_ || defined _SAPB_ || defined _OLDOFFICE01_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _PBKDF2_MD5_ || defined _KRB5TGS_ + +#define MD5_F_S(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define MD5_G_S(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) +#define MD5_H_S(x,y,z) ((x) ^ (y) ^ (z)) +#define MD5_I_S(x,y,z) ((y) ^ ((x) | ~(z))) + +#ifdef IS_NV +#define MD5_F(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define MD5_G(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) +#define MD5_H(x,y,z) ((x) ^ (y) ^ (z)) +#define MD5_I(x,y,z) ((y) ^ ((x) | ~(z))) +#define MD5_Fo(x,y,z) (MD5_F((x), (y), (z))) +#define MD5_Go(x,y,z) (MD5_G((x), (y), (z))) +#endif + +#ifdef IS_AMD +#define MD5_F(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define MD5_G(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) +#define MD5_H(x,y,z) ((x) ^ (y) ^ (z)) +#define MD5_I(x,y,z) (bitselect (0xffffffffU, (x), (z)) ^ (y)) +#define MD5_Fo(x,y,z) (bitselect ((z), (y), (x))) +#define MD5_Go(x,y,z) (bitselect ((y), (x), (z))) +#endif + +#ifdef IS_GENERIC +#define MD5_F(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define MD5_G(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) +#define MD5_H(x,y,z) ((x) ^ (y) ^ (z)) +#define MD5_I(x,y,z) ((y) ^ ((x) | ~(z))) +#define MD5_Fo(x,y,z) (MD5_F((x), (y), (z))) +#define MD5_Go(x,y,z) (MD5_G((x), (y), (z))) +#endif + +#define MD5_STEP_S(f,a,b,c,d,x,K,s) \ +{ \ + a += K; \ + a += x; \ + a += f (b, c, d); \ + a = rotl32_S (a, s); \ + a += b; \ +} + +#define MD5_STEP(f,a,b,c,d,x,K,s) \ +{ \ + a += K; \ + a += x; \ + a += f (b, c, d); \ + a = rotl32 (a, s); \ + a += b; \ +} + +#define MD5_STEP0(f,a,b,c,d,K,s) \ +{ \ + a += K; \ + a += f (b, c, d); \ + a = rotl32 (a, s); \ + a += b; \ +} +#endif + +#if defined _SHA1_ || defined _SAPG_ || defined _OFFICE2007_ || defined _OFFICE2010_ || defined _OLDOFFICE34_ || defined _ANDROIDFDE_ || defined _DCC2_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _PSAFE2_ || defined _LOTUS8_ || defined _PBKDF2_SHA1_ || defined _RAR3_ || defined _SHA256_SHA1_ || defined _ZIP2_ + +#ifdef IS_NV +#define SHA1_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define SHA1_F1(x,y,z) ((x) ^ (y) ^ (z)) +#define SHA1_F2(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) +#define SHA1_F0o(x,y,z) (SHA1_F0 ((x), (y), (z))) +#define SHA1_F2o(x,y,z) (SHA1_F2 ((x), (y), (z))) +#endif + +#ifdef IS_AMD +#define SHA1_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define SHA1_F1(x,y,z) ((x) ^ (y) ^ (z)) +#define SHA1_F2(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) +#define SHA1_F0o(x,y,z) (bitselect ((z), (y), (x))) +#define SHA1_F2o(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) +#endif + +#ifdef IS_GENERIC +#define SHA1_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define SHA1_F1(x,y,z) ((x) ^ (y) ^ (z)) +#define SHA1_F2(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) +#define SHA1_F0o(x,y,z) (SHA1_F0 ((x), (y), (z))) +#define SHA1_F2o(x,y,z) (SHA1_F2 ((x), (y), (z))) +#endif + +#define SHA1_STEP_S(f,a,b,c,d,e,x) \ +{ \ + e += K; \ + e += x; \ + e += f (b, c, d); \ + e += rotl32_S (a, 5u); \ + b = rotl32_S (b, 30u); \ +} + +#define SHA1_STEP(f,a,b,c,d,e,x) \ +{ \ + e += K; \ + e += x; \ + e += f (b, c, d); \ + e += rotl32 (a, 5u); \ + b = rotl32 (b, 30u); \ +} + +#define SHA1_STEP0(f,a,b,c,d,e,x) \ +{ \ + e += K; \ + e += f (b, c, d); \ + e += rotl32 (a, 5u); \ + b = rotl32 (b, 30u); \ +} + +#define SHA1_STEPX(f,a,b,c,d,e,x) \ +{ \ + e += x; \ + e += f (b, c, d); \ + e += rotl32 (a, 5u); \ + b = rotl32 (b, 30u); \ +} + +#define SHA1_STEP_PE(f,a,b,c,d,e,x) \ +{ \ + e += x; \ + e += f (b, c, d); \ + e += rotl32 (a, 5u); \ +} + +#define SHA1_STEP_PB(f,a,b,c,d,e,x) \ +{ \ + e += K; \ + b = rotl32 (b, 30u); \ +} +#endif + +#if defined _SHA256_ || defined _PDF17L8_ || defined _SEVEN_ZIP_ || defined _ANDROIDFDE_ || defined _CLOUDKEY_ || defined _SCRYPT_ || defined _PBKDF2_SHA256_ || defined _SHA256_SHA1_ || defined _MS_DRSR_ || defined _ANDROIDFDE_SAMSUNG_ || defined _RAR5_ || defined _KEEPASS_ + +#define SHIFT_RIGHT_32(x,n) ((x) >> (n)) + +#define SHA256_S0_S(x) (rotl32_S ((x), 25u) ^ rotl32_S ((x), 14u) ^ SHIFT_RIGHT_32 ((x), 3u)) +#define SHA256_S1_S(x) (rotl32_S ((x), 15u) ^ rotl32_S ((x), 13u) ^ SHIFT_RIGHT_32 ((x), 10u)) +#define SHA256_S2_S(x) (rotl32_S ((x), 30u) ^ rotl32_S ((x), 19u) ^ rotl32_S ((x), 10u)) +#define SHA256_S3_S(x) (rotl32_S ((x), 26u) ^ rotl32_S ((x), 21u) ^ rotl32_S ((x), 7u)) + +#define SHA256_S0(x) (rotl32 ((x), 25u) ^ rotl32 ((x), 14u) ^ SHIFT_RIGHT_32 ((x), 3u)) +#define SHA256_S1(x) (rotl32 ((x), 15u) ^ rotl32 ((x), 13u) ^ SHIFT_RIGHT_32 ((x), 10u)) +#define SHA256_S2(x) (rotl32 ((x), 30u) ^ rotl32 ((x), 19u) ^ rotl32 ((x), 10u)) +#define SHA256_S3(x) (rotl32 ((x), 26u) ^ rotl32 ((x), 21u) ^ rotl32 ((x), 7u)) + +#ifdef IS_NV +#define SHA256_F0(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) +#define SHA256_F1(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define SHA256_F0o(x,y,z) (SHA256_F0 ((x), (y), (z))) +#define SHA256_F1o(x,y,z) (SHA256_F1 ((x), (y), (z))) +#endif + +#ifdef IS_AMD +#define SHA256_F0(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) +#define SHA256_F1(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define SHA256_F0o(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) +#define SHA256_F1o(x,y,z) (bitselect ((z), (y), (x))) +#endif + +#ifdef IS_GENERIC +#define SHA256_F0(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) +#define SHA256_F1(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define SHA256_F0o(x,y,z) (SHA256_F0 ((x), (y), (z))) +#define SHA256_F1o(x,y,z) (SHA256_F1 ((x), (y), (z))) +#endif + +#define SHA256_STEP_S(F0,F1,a,b,c,d,e,f,g,h,x,K) \ +{ \ + h += K; \ + h += x; \ + h += SHA256_S3_S (e); \ + h += F1 (e,f,g); \ + d += h; \ + h += SHA256_S2_S (a); \ + h += F0 (a,b,c); \ +} + +#define SHA256_EXPAND_S(x,y,z,w) (SHA256_S1_S (x) + y + SHA256_S0_S (z) + w) + +#define SHA256_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K) \ +{ \ + h += K; \ + h += x; \ + h += SHA256_S3 (e); \ + h += F1 (e,f,g); \ + d += h; \ + h += SHA256_S2 (a); \ + h += F0 (a,b,c); \ +} + +#define SHA256_EXPAND(x,y,z,w) (SHA256_S1 (x) + y + SHA256_S0 (z) + w) + +#endif + +#if defined _SHA384_ || defined _PDF17L8_ + +#define SHIFT_RIGHT_64(x,n) ((x) >> (n)) + +#define SHA384_S0(x) (rotr64 ((x), 28) ^ rotr64 ((x), 34) ^ rotr64 ((x), 39)) +#define SHA384_S1(x) (rotr64 ((x), 14) ^ rotr64 ((x), 18) ^ rotr64 ((x), 41)) +#define SHA384_S2(x) (rotr64 ((x), 1) ^ rotr64 ((x), 8) ^ SHIFT_RIGHT_64 ((x), 7)) +#define SHA384_S3(x) (rotr64 ((x), 19) ^ rotr64 ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6)) + +#define SHA384_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define SHA384_F1(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) + +#ifdef IS_NV +#define SHA384_F0o(x,y,z) (SHA384_F0 ((x), (y), (z))) +#define SHA384_F1o(x,y,z) (SHA384_F1 ((x), (y), (z))) +#endif + +#ifdef IS_AMD +#define SHA384_F0o(x,y,z) (bitselect ((z), (y), (x))) +#define SHA384_F1o(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) +#endif + +#ifdef IS_GENERIC +#define SHA384_F0o(x,y,z) (SHA384_F0 ((x), (y), (z))) +#define SHA384_F1o(x,y,z) (SHA384_F1 ((x), (y), (z))) +#endif + +#define SHA384_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K) \ +{ \ + h += K; \ + h += x; \ + h += SHA384_S1 (e); \ + h += F0 (e, f, g); \ + d += h; \ + h += SHA384_S0 (a); \ + h += F1 (a, b, c); \ +} + +#define SHA384_EXPAND(x,y,z,w) (SHA384_S3 (x) + y + SHA384_S2 (z) + w) +#endif + +#if defined _SHA512_ || defined _CLOUDKEY_ || defined _OFFICE2013_ || defined _PDF17L8_ || defined _PBKDF2_SHA512_ + +#define SHIFT_RIGHT_64(x,n) ((x) >> (n)) + +#define SHA512_S0_S(x) (rotr64_S ((x), 28) ^ rotr64_S ((x), 34) ^ rotr64_S ((x), 39)) +#define SHA512_S1_S(x) (rotr64_S ((x), 14) ^ rotr64_S ((x), 18) ^ rotr64_S ((x), 41)) +#define SHA512_S2_S(x) (rotr64_S ((x), 1) ^ rotr64_S ((x), 8) ^ SHIFT_RIGHT_64 ((x), 7)) +#define SHA512_S3_S(x) (rotr64_S ((x), 19) ^ rotr64_S ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6)) + +#define SHA512_S0(x) (rotr64 ((x), 28) ^ rotr64 ((x), 34) ^ rotr64 ((x), 39)) +#define SHA512_S1(x) (rotr64 ((x), 14) ^ rotr64 ((x), 18) ^ rotr64 ((x), 41)) +#define SHA512_S2(x) (rotr64 ((x), 1) ^ rotr64 ((x), 8) ^ SHIFT_RIGHT_64 ((x), 7)) +#define SHA512_S3(x) (rotr64 ((x), 19) ^ rotr64 ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6)) + +#define SHA512_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) +#define SHA512_F1(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) + +#ifdef IS_NV +#define SHA512_F0o(x,y,z) (SHA512_F0 ((x), (y), (z))) +#define SHA512_F1o(x,y,z) (SHA512_F1 ((x), (y), (z))) +#endif + +#ifdef IS_AMD +#define SHA512_F0o(x,y,z) (bitselect ((z), (y), (x))) +#define SHA512_F1o(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) +#endif + +#ifdef IS_GENERIC +#define SHA512_F0o(x,y,z) (SHA512_F0 ((x), (y), (z))) +#define SHA512_F1o(x,y,z) (SHA512_F1 ((x), (y), (z))) +#endif + +#define SHA512_STEP_S(F0,F1,a,b,c,d,e,f,g,h,x,K) \ +{ \ + h += K; \ + h += x; \ + h += SHA512_S1_S (e); \ + h += F0 (e, f, g); \ + d += h; \ + h += SHA512_S0_S (a); \ + h += F1 (a, b, c); \ +} + +#define SHA512_EXPAND_S(x,y,z,w) (SHA512_S3_S (x) + y + SHA512_S2_S (z) + w) + +#define SHA512_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K) \ +{ \ + h += K; \ + h += x; \ + h += SHA512_S1 (e); \ + h += F0 (e, f, g); \ + d += h; \ + h += SHA512_S0 (a); \ + h += F1 (a, b, c); \ +} + +#define SHA512_EXPAND(x,y,z,w) (SHA512_S3 (x) + y + SHA512_S2 (z) + w) +#endif + +#ifdef _RIPEMD160_ + +#ifdef IS_NV +#define RIPEMD160_F(x,y,z) ((x) ^ (y) ^ (z)) +#define RIPEMD160_G(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */ +#define RIPEMD160_H(x,y,z) (((x) | ~(y)) ^ (z)) +#define RIPEMD160_I(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */ +#define RIPEMD160_J(x,y,z) ((x) ^ ((y) | ~(z))) +#define RIPEMD160_Go(x,y,z) (RIPEMD160_G ((x), (y), (z))) +#define RIPEMD160_Io(x,y,z) (RIPEMD160_I ((x), (y), (z))) +#endif + +#ifdef IS_AMD +#define RIPEMD160_F(x,y,z) ((x) ^ (y) ^ (z)) +#define RIPEMD160_G(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */ +#define RIPEMD160_H(x,y,z) (((x) | ~(y)) ^ (z)) +#define RIPEMD160_I(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */ +#define RIPEMD160_J(x,y,z) ((x) ^ ((y) | ~(z))) +#define RIPEMD160_Go(x,y,z) (bitselect ((z), (y), (x))) +#define RIPEMD160_Io(x,y,z) (bitselect ((y), (x), (z))) +#endif + +#ifdef IS_GENERIC +#define RIPEMD160_F(x,y,z) ((x) ^ (y) ^ (z)) +#define RIPEMD160_G(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */ +#define RIPEMD160_H(x,y,z) (((x) | ~(y)) ^ (z)) +#define RIPEMD160_I(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */ +#define RIPEMD160_J(x,y,z) ((x) ^ ((y) | ~(z))) +#define RIPEMD160_Go(x,y,z) (RIPEMD160_G ((x), (y), (z))) +#define RIPEMD160_Io(x,y,z) (RIPEMD160_I ((x), (y), (z))) +#endif + +#define RIPEMD160_STEP(f,a,b,c,d,e,x,K,s) \ +{ \ + a += K; \ + a += x; \ + a += f (b, c, d); \ + a = rotl32 (a, s); \ + a += e; \ + c = rotl32 (c, 10u); \ +} + +#define ROTATE_LEFT_WORKAROUND_BUG(a,n) ((a << n) | (a >> (32 - n))) + +#define RIPEMD160_STEP_WORKAROUND_BUG(f,a,b,c,d,e,x,K,s) \ +{ \ + a += K; \ + a += x; \ + a += f (b, c, d); \ + a = ROTATE_LEFT_WORKAROUND_BUG (a, s); \ + a += e; \ + c = rotl32 (c, 10u); \ +} + +#endif diff --git a/OpenCL/inc_rp.cl b/OpenCL/inc_rp.cl new file mode 100644 index 0000000..3a93e19 --- /dev/null +++ b/OpenCL/inc_rp.cl @@ -0,0 +1,4357 @@ +/** + * Authors.....: Jens Steube + * magnum + * + * License.....: MIT + */ + +inline u32 apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len); +inline u32 apply_rules (const __global u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len); +inline u32x apply_rules_vect (const u32 pw_buf0[4], const u32 pw_buf1[4], const u32 pw_len, const __global kernel_rule_t *rules_buf, const u32 il_pos, u32x w0[4], u32x w1[4]); + +inline u32 generate_cmask (u32 buf) +{ + const u32 rmask = ((buf & 0x40404040) >> 1) + & ~((buf & 0x80808080) >> 2); + + const u32 hmask = (buf & 0x1f1f1f1f) + 0x05050505; + const u32 lmask = (buf & 0x1f1f1f1f) + 0x1f1f1f1f; + + return rmask & ~hmask & lmask; +} + +inline void truncate_right (u32 w0[4], u32 w1[4], const u32 len) +{ + const u32 tmp = (1 << ((len % 4) * 8)) - 1; + + switch (len / 4) + { + case 0: w0[0] &= tmp; + w0[1] = 0; + w0[2] = 0; + w0[3] = 0; + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + break; + case 1: w0[1] &= tmp; + w0[2] = 0; + w0[3] = 0; + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + break; + case 2: w0[2] &= tmp; + w0[3] = 0; + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + break; + case 3: w0[3] &= tmp; + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + break; + case 4: w1[0] &= tmp; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + break; + case 5: w1[1] &= tmp; + w1[2] = 0; + w1[3] = 0; + break; + case 6: w1[2] &= tmp; + w1[3] = 0; + break; + case 7: w1[3] &= tmp; + break; + } +} + +inline void truncate_left (u32 w0[4], u32 w1[4], const u32 len) +{ + const u32 tmp = ~((1 << ((len % 4) * 8)) - 1); + + switch (len / 4) + { + case 0: w0[0] &= tmp; + break; + case 1: w0[0] = 0; + w0[1] &= tmp; + break; + case 2: w0[0] = 0; + w0[1] = 0; + w0[2] &= tmp; + break; + case 3: w0[0] = 0; + w0[1] = 0; + w0[2] = 0; + w0[3] &= tmp; + break; + case 4: w0[0] = 0; + w0[1] = 0; + w0[2] = 0; + w0[3] = 0; + w1[0] &= tmp; + break; + case 5: w0[0] = 0; + w0[1] = 0; + w0[2] = 0; + w0[3] = 0; + w1[0] = 0; + w1[1] &= tmp; + break; + case 6: w0[0] = 0; + w0[1] = 0; + w0[2] = 0; + w0[3] = 0; + w1[0] = 0; + w1[1] = 0; + w1[2] &= tmp; + break; + case 7: w0[0] = 0; + w0[1] = 0; + w0[2] = 0; + w0[3] = 0; + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] &= tmp; + break; + } +} + +inline void lshift_block (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4]) +{ + #ifdef IS_NV + out0[0] = __byte_perm_S (in0[0], in0[1], 0x4321); + out0[1] = __byte_perm_S (in0[1], in0[2], 0x4321); + out0[2] = __byte_perm_S (in0[2], in0[3], 0x4321); + out0[3] = __byte_perm_S (in0[3], in1[0], 0x4321); + out1[0] = __byte_perm_S (in1[0], in1[1], 0x4321); + out1[1] = __byte_perm_S (in1[1], in1[2], 0x4321); + out1[2] = __byte_perm_S (in1[2], in1[3], 0x4321); + out1[3] = __byte_perm_S (in1[3], 0, 0x4321); + #endif + + #if defined IS_AMD || defined IS_GENERIC + out0[0] = amd_bytealign_S (in0[1], in0[0], 1); + out0[1] = amd_bytealign_S (in0[2], in0[1], 1); + out0[2] = amd_bytealign_S (in0[3], in0[2], 1); + out0[3] = amd_bytealign_S (in1[0], in0[3], 1); + out1[0] = amd_bytealign_S (in1[1], in1[0], 1); + out1[1] = amd_bytealign_S (in1[2], in1[1], 1); + out1[2] = amd_bytealign_S (in1[3], in1[2], 1); + out1[3] = amd_bytealign_S ( 0, in1[3], 1); + #endif +} + +inline void rshift_block (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4]) +{ + #ifdef IS_NV + out1[3] = __byte_perm_S (in1[2], in1[3], 0x6543); + out1[2] = __byte_perm_S (in1[1], in1[2], 0x6543); + out1[1] = __byte_perm_S (in1[0], in1[1], 0x6543); + out1[0] = __byte_perm_S (in0[3], in1[0], 0x6543); + out0[3] = __byte_perm_S (in0[2], in0[3], 0x6543); + out0[2] = __byte_perm_S (in0[1], in0[2], 0x6543); + out0[1] = __byte_perm_S (in0[0], in0[1], 0x6543); + out0[0] = __byte_perm_S ( 0, in0[0], 0x6543); + #endif + + #if defined IS_AMD || defined IS_GENERIC + out1[3] = amd_bytealign_S (in1[3], in1[2], 3); + out1[2] = amd_bytealign_S (in1[2], in1[1], 3); + out1[1] = amd_bytealign_S (in1[1], in1[0], 3); + out1[0] = amd_bytealign_S (in1[0], in0[3], 3); + out0[3] = amd_bytealign_S (in0[3], in0[2], 3); + out0[2] = amd_bytealign_S (in0[2], in0[1], 3); + out0[1] = amd_bytealign_S (in0[1], in0[0], 3); + out0[0] = amd_bytealign_S (in0[0], 0, 3); + #endif +} + +inline void lshift_block_N (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4], const u32 num) +{ + #ifdef IS_NV + switch (num) + { + case 0: out0[0] = in0[0]; + out0[1] = in0[1]; + out0[2] = in0[2]; + out0[3] = in0[3]; + out1[0] = in1[0]; + out1[1] = in1[1]; + out1[2] = in1[2]; + out1[3] = in1[3]; + break; + case 1: out0[0] = __byte_perm_S (in0[0], in0[1], 0x4321); + out0[1] = __byte_perm_S (in0[1], in0[2], 0x4321); + out0[2] = __byte_perm_S (in0[2], in0[3], 0x4321); + out0[3] = __byte_perm_S (in0[3], in1[0], 0x4321); + out1[0] = __byte_perm_S (in1[0], in1[1], 0x4321); + out1[1] = __byte_perm_S (in1[1], in1[2], 0x4321); + out1[2] = __byte_perm_S (in1[2], in1[3], 0x4321); + out1[3] = __byte_perm_S (in1[3], 0, 0x4321); + break; + case 2: out0[0] = __byte_perm_S (in0[0], in0[1], 0x5432); + out0[1] = __byte_perm_S (in0[1], in0[2], 0x5432); + out0[2] = __byte_perm_S (in0[2], in0[3], 0x5432); + out0[3] = __byte_perm_S (in0[3], in1[0], 0x5432); + out1[0] = __byte_perm_S (in1[0], in1[1], 0x5432); + out1[1] = __byte_perm_S (in1[1], in1[2], 0x5432); + out1[2] = __byte_perm_S (in1[2], in1[3], 0x5432); + out1[3] = __byte_perm_S (in1[3], 0, 0x5432); + break; + case 3: out0[0] = __byte_perm_S (in0[0], in0[1], 0x6543); + out0[1] = __byte_perm_S (in0[1], in0[2], 0x6543); + out0[2] = __byte_perm_S (in0[2], in0[3], 0x6543); + out0[3] = __byte_perm_S (in0[3], in1[0], 0x6543); + out1[0] = __byte_perm_S (in1[0], in1[1], 0x6543); + out1[1] = __byte_perm_S (in1[1], in1[2], 0x6543); + out1[2] = __byte_perm_S (in1[2], in1[3], 0x6543); + out1[3] = __byte_perm_S (in1[3], 0, 0x6543); + break; + case 4: out0[0] = in0[1]; + out0[1] = in0[2]; + out0[2] = in0[3]; + out0[3] = in1[0]; + out1[0] = in1[1]; + out1[1] = in1[2]; + out1[2] = in1[3]; + out1[3] = 0; + break; + case 5: out0[0] = __byte_perm_S (in0[1], in0[2], 0x4321); + out0[1] = __byte_perm_S (in0[2], in0[3], 0x4321); + out0[2] = __byte_perm_S (in0[3], in1[0], 0x4321); + out0[3] = __byte_perm_S (in1[0], in1[1], 0x4321); + out1[0] = __byte_perm_S (in1[1], in1[2], 0x4321); + out1[1] = __byte_perm_S (in1[2], in1[3], 0x4321); + out1[2] = __byte_perm_S (in1[3], 0, 0x4321); + out1[3] = 0; + break; + case 6: out0[0] = __byte_perm_S (in0[1], in0[2], 0x5432); + out0[1] = __byte_perm_S (in0[2], in0[3], 0x5432); + out0[2] = __byte_perm_S (in0[3], in1[0], 0x5432); + out0[3] = __byte_perm_S (in1[0], in1[1], 0x5432); + out1[0] = __byte_perm_S (in1[1], in1[2], 0x5432); + out1[1] = __byte_perm_S (in1[2], in1[3], 0x5432); + out1[2] = __byte_perm_S (in1[3], 0, 0x5432); + out1[3] = 0; + break; + case 7: out0[0] = __byte_perm_S (in0[1], in0[2], 0x6543); + out0[1] = __byte_perm_S (in0[2], in0[3], 0x6543); + out0[2] = __byte_perm_S (in0[3], in1[0], 0x6543); + out0[3] = __byte_perm_S (in1[0], in1[1], 0x6543); + out1[0] = __byte_perm_S (in1[1], in1[2], 0x6543); + out1[1] = __byte_perm_S (in1[2], in1[3], 0x6543); + out1[2] = __byte_perm_S (in1[3], 0, 0x6543); + out1[3] = 0; + break; + case 8: out0[0] = in0[2]; + out0[1] = in0[3]; + out0[2] = in1[0]; + out0[3] = in1[1]; + out1[0] = in1[2]; + out1[1] = in1[3]; + out1[2] = 0; + out1[3] = 0; + break; + case 9: out0[0] = __byte_perm_S (in0[2], in0[3], 0x4321); + out0[1] = __byte_perm_S (in0[3], in1[0], 0x4321); + out0[2] = __byte_perm_S (in1[0], in1[1], 0x4321); + out0[3] = __byte_perm_S (in1[1], in1[2], 0x4321); + out1[0] = __byte_perm_S (in1[2], in1[3], 0x4321); + out1[1] = __byte_perm_S (in1[3], 0, 0x4321); + out1[2] = 0; + out1[3] = 0; + break; + case 10: out0[0] = __byte_perm_S (in0[2], in0[3], 0x5432); + out0[1] = __byte_perm_S (in0[3], in1[0], 0x5432); + out0[2] = __byte_perm_S (in1[0], in1[1], 0x5432); + out0[3] = __byte_perm_S (in1[1], in1[2], 0x5432); + out1[0] = __byte_perm_S (in1[2], in1[3], 0x5432); + out1[1] = __byte_perm_S (in1[3], 0, 0x5432); + out1[2] = 0; + out1[3] = 0; + break; + case 11: out0[0] = __byte_perm_S (in0[2], in0[3], 0x6543); + out0[1] = __byte_perm_S (in0[3], in1[0], 0x6543); + out0[2] = __byte_perm_S (in1[0], in1[1], 0x6543); + out0[3] = __byte_perm_S (in1[1], in1[2], 0x6543); + out1[0] = __byte_perm_S (in1[2], in1[3], 0x6543); + out1[1] = __byte_perm_S (in1[3], 0, 0x6543); + out1[2] = 0; + out1[3] = 0; + break; + case 12: out0[0] = in0[3]; + out0[1] = in1[0]; + out0[2] = in1[1]; + out0[3] = in1[2]; + out1[0] = in1[3]; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 13: + out0[0] = __byte_perm_S (in0[3], in1[0], 0x4321); + out0[1] = __byte_perm_S (in1[0], in1[1], 0x4321); + out0[2] = __byte_perm_S (in1[1], in1[2], 0x4321); + out0[3] = __byte_perm_S (in1[2], in1[3], 0x4321); + out1[0] = __byte_perm_S (in1[3], 0, 0x4321); + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 14: out0[0] = __byte_perm_S (in0[3], in1[0], 0x5432); + out0[1] = __byte_perm_S (in1[0], in1[1], 0x5432); + out0[2] = __byte_perm_S (in1[1], in1[2], 0x5432); + out0[3] = __byte_perm_S (in1[2], in1[3], 0x5432); + out1[0] = __byte_perm_S (in1[3], 0, 0x5432); + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 15: out0[0] = __byte_perm_S (in0[3], in1[0], 0x6543); + out0[1] = __byte_perm_S (in1[0], in1[1], 0x6543); + out0[2] = __byte_perm_S (in1[1], in1[2], 0x6543); + out0[3] = __byte_perm_S (in1[2], in1[3], 0x6543); + out1[0] = __byte_perm_S (in1[3], 0, 0x6543); + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 16: out0[0] = in1[0]; + out0[1] = in1[1]; + out0[2] = in1[2]; + out0[3] = in1[3]; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 17: out0[0] = __byte_perm_S (in1[0], in1[1], 0x4321); + out0[1] = __byte_perm_S (in1[1], in1[2], 0x4321); + out0[2] = __byte_perm_S (in1[2], in1[3], 0x4321); + out0[3] = __byte_perm_S (in1[3], 0, 0x4321); + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 18: out0[0] = __byte_perm_S (in1[0], in1[1], 0x5432); + out0[1] = __byte_perm_S (in1[1], in1[2], 0x5432); + out0[2] = __byte_perm_S (in1[2], in1[3], 0x5432); + out0[3] = __byte_perm_S (in1[3], 0, 0x5432); + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 19: out0[0] = __byte_perm_S (in1[0], in1[1], 0x6543); + out0[1] = __byte_perm_S (in1[1], in1[2], 0x6543); + out0[2] = __byte_perm_S (in1[2], in1[3], 0x6543); + out0[3] = __byte_perm_S (in1[3], 0, 0x6543); + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 20: out0[0] = in1[1]; + out0[1] = in1[2]; + out0[2] = in1[3]; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 21: out0[0] = __byte_perm_S (in1[1], in1[2], 0x4321); + out0[1] = __byte_perm_S (in1[2], in1[3], 0x4321); + out0[2] = __byte_perm_S (in1[3], 0, 0x4321); + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 22: out0[0] = __byte_perm_S (in1[1], in1[2], 0x5432); + out0[1] = __byte_perm_S (in1[2], in1[3], 0x5432); + out0[2] = __byte_perm_S (in1[3], 0, 0x5432); + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 23: out0[0] = __byte_perm_S (in1[1], in1[2], 0x6543); + out0[1] = __byte_perm_S (in1[2], in1[3], 0x6543); + out0[2] = __byte_perm_S (in1[3], 0, 0x6543); + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 24: out0[0] = in1[2]; + out0[1] = in1[3]; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 25: out0[0] = __byte_perm_S (in1[2], in1[3], 0x4321); + out0[1] = __byte_perm_S (in1[3], 0, 0x4321); + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 26: out0[0] = __byte_perm_S (in1[2], in1[3], 0x5432); + out0[1] = __byte_perm_S (in1[3], 0, 0x5432); + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 27: out0[0] = __byte_perm_S (in1[2], in1[3], 0x6543); + out0[1] = __byte_perm_S (in1[3], 0, 0x6543); + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 28: out0[0] = in1[3]; + out0[1] = 0; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 29: out0[0] = __byte_perm_S (in1[3], 0, 0x4321); + out0[1] = 0; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 30: out0[0] = __byte_perm_S (in1[3], 0, 0x5432); + out0[1] = 0; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 31: out0[0] = __byte_perm_S (in1[3], 0, 0x6543); + out0[1] = 0; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + } + #endif + + #if defined IS_AMD || defined IS_GENERIC + switch (num) + { + case 0: out0[0] = in0[0]; + out0[1] = in0[1]; + out0[2] = in0[2]; + out0[3] = in0[3]; + out1[0] = in1[0]; + out1[1] = in1[1]; + out1[2] = in1[2]; + out1[3] = in1[3]; + break; + case 1: out0[0] = amd_bytealign_S (in0[1], in0[0], 1); + out0[1] = amd_bytealign_S (in0[2], in0[1], 1); + out0[2] = amd_bytealign_S (in0[3], in0[2], 1); + out0[3] = amd_bytealign_S (in1[0], in0[3], 1); + out1[0] = amd_bytealign_S (in1[1], in1[0], 1); + out1[1] = amd_bytealign_S (in1[2], in1[1], 1); + out1[2] = amd_bytealign_S (in1[3], in1[2], 1); + out1[3] = amd_bytealign_S ( 0, in1[3], 1); + break; + case 2: out0[0] = amd_bytealign_S (in0[1], in0[0], 2); + out0[1] = amd_bytealign_S (in0[2], in0[1], 2); + out0[2] = amd_bytealign_S (in0[3], in0[2], 2); + out0[3] = amd_bytealign_S (in1[0], in0[3], 2); + out1[0] = amd_bytealign_S (in1[1], in1[0], 2); + out1[1] = amd_bytealign_S (in1[2], in1[1], 2); + out1[2] = amd_bytealign_S (in1[3], in1[2], 2); + out1[3] = amd_bytealign_S ( 0, in1[3], 2); + break; + case 3: out0[0] = amd_bytealign_S (in0[1], in0[0], 3); + out0[1] = amd_bytealign_S (in0[2], in0[1], 3); + out0[2] = amd_bytealign_S (in0[3], in0[2], 3); + out0[3] = amd_bytealign_S (in1[0], in0[3], 3); + out1[0] = amd_bytealign_S (in1[1], in1[0], 3); + out1[1] = amd_bytealign_S (in1[2], in1[1], 3); + out1[2] = amd_bytealign_S (in1[3], in1[2], 3); + out1[3] = amd_bytealign_S ( 0, in1[3], 3); + break; + case 4: out0[0] = in0[1]; + out0[1] = in0[2]; + out0[2] = in0[3]; + out0[3] = in1[0]; + out1[0] = in1[1]; + out1[1] = in1[2]; + out1[2] = in1[3]; + out1[3] = 0; + break; + case 5: out0[0] = amd_bytealign_S (in0[2], in0[1], 1); + out0[1] = amd_bytealign_S (in0[3], in0[2], 1); + out0[2] = amd_bytealign_S (in1[0], in0[3], 1); + out0[3] = amd_bytealign_S (in1[1], in1[0], 1); + out1[0] = amd_bytealign_S (in1[2], in1[1], 1); + out1[1] = amd_bytealign_S (in1[3], in1[2], 1); + out1[2] = amd_bytealign_S ( 0, in1[3], 1); + out1[3] = 0; + break; + case 6: out0[0] = amd_bytealign_S (in0[2], in0[1], 2); + out0[1] = amd_bytealign_S (in0[3], in0[2], 2); + out0[2] = amd_bytealign_S (in1[0], in0[3], 2); + out0[3] = amd_bytealign_S (in1[1], in1[0], 2); + out1[0] = amd_bytealign_S (in1[2], in1[1], 2); + out1[1] = amd_bytealign_S (in1[3], in1[2], 2); + out1[2] = amd_bytealign_S ( 0, in1[3], 2); + out1[3] = 0; + break; + case 7: out0[0] = amd_bytealign_S (in0[2], in0[1], 3); + out0[1] = amd_bytealign_S (in0[3], in0[2], 3); + out0[2] = amd_bytealign_S (in1[0], in0[3], 3); + out0[3] = amd_bytealign_S (in1[1], in1[0], 3); + out1[0] = amd_bytealign_S (in1[2], in1[1], 3); + out1[1] = amd_bytealign_S (in1[3], in1[2], 3); + out1[2] = amd_bytealign_S ( 0, in1[3], 3); + out1[3] = 0; + break; + case 8: out0[0] = in0[2]; + out0[1] = in0[3]; + out0[2] = in1[0]; + out0[3] = in1[1]; + out1[0] = in1[2]; + out1[1] = in1[3]; + out1[2] = 0; + out1[3] = 0; + break; + case 9: out0[0] = amd_bytealign_S (in0[3], in0[2], 1); + out0[1] = amd_bytealign_S (in1[0], in0[3], 1); + out0[2] = amd_bytealign_S (in1[1], in1[0], 1); + out0[3] = amd_bytealign_S (in1[2], in1[1], 1); + out1[0] = amd_bytealign_S (in1[3], in1[2], 1); + out1[1] = amd_bytealign_S ( 0, in1[3], 1); + out1[2] = 0; + out1[3] = 0; + break; + case 10: out0[0] = amd_bytealign_S (in0[3], in0[2], 2); + out0[1] = amd_bytealign_S (in1[0], in0[3], 2); + out0[2] = amd_bytealign_S (in1[1], in1[0], 2); + out0[3] = amd_bytealign_S (in1[2], in1[1], 2); + out1[0] = amd_bytealign_S (in1[3], in1[2], 2); + out1[1] = amd_bytealign_S ( 0, in1[3], 2); + out1[2] = 0; + out1[3] = 0; + break; + case 11: out0[0] = amd_bytealign_S (in0[3], in0[2], 3); + out0[1] = amd_bytealign_S (in1[0], in0[3], 3); + out0[2] = amd_bytealign_S (in1[1], in1[0], 3); + out0[3] = amd_bytealign_S (in1[2], in1[1], 3); + out1[0] = amd_bytealign_S (in1[3], in1[2], 3); + out1[1] = amd_bytealign_S ( 0, in1[3], 3); + out1[2] = 0; + out1[3] = 0; + break; + case 12: out0[0] = in0[3]; + out0[1] = in1[0]; + out0[2] = in1[1]; + out0[3] = in1[2]; + out1[0] = in1[3]; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 13: out0[0] = amd_bytealign_S (in1[0], in0[3], 1); + out0[1] = amd_bytealign_S (in1[1], in1[0], 1); + out0[2] = amd_bytealign_S (in1[2], in1[1], 1); + out0[3] = amd_bytealign_S (in1[3], in1[2], 1); + out1[0] = amd_bytealign_S ( 0, in1[3], 1); + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 14: out0[0] = amd_bytealign_S (in1[0], in0[3], 2); + out0[1] = amd_bytealign_S (in1[1], in1[0], 2); + out0[2] = amd_bytealign_S (in1[2], in1[1], 2); + out0[3] = amd_bytealign_S (in1[3], in1[2], 2); + out1[0] = amd_bytealign_S ( 0, in1[3], 2); + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 15: out0[0] = amd_bytealign_S (in1[0], in0[3], 3); + out0[1] = amd_bytealign_S (in1[1], in1[0], 3); + out0[2] = amd_bytealign_S (in1[2], in1[1], 3); + out0[3] = amd_bytealign_S (in1[3], in1[2], 3); + out1[0] = amd_bytealign_S ( 0, in1[3], 3); + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 16: out0[0] = in1[0]; + out0[1] = in1[1]; + out0[2] = in1[2]; + out0[3] = in1[3]; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 17: out0[0] = amd_bytealign_S (in1[1], in1[0], 1); + out0[1] = amd_bytealign_S (in1[2], in1[1], 1); + out0[2] = amd_bytealign_S (in1[3], in1[2], 1); + out0[3] = amd_bytealign_S ( 0, in1[3], 1); + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 18: out0[0] = amd_bytealign_S (in1[1], in1[0], 2); + out0[1] = amd_bytealign_S (in1[2], in1[1], 2); + out0[2] = amd_bytealign_S (in1[3], in1[2], 2); + out0[3] = amd_bytealign_S ( 0, in1[3], 2); + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 19: out0[0] = amd_bytealign_S (in1[1], in1[0], 3); + out0[1] = amd_bytealign_S (in1[2], in1[1], 3); + out0[2] = amd_bytealign_S (in1[3], in1[2], 3); + out0[3] = amd_bytealign_S ( 0, in1[3], 3); + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 20: out0[0] = in1[1]; + out0[1] = in1[2]; + out0[2] = in1[3]; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 21: out0[0] = amd_bytealign_S (in1[2], in1[1], 1); + out0[1] = amd_bytealign_S (in1[3], in1[2], 1); + out0[2] = amd_bytealign_S ( 0, in1[3], 1); + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 22: out0[0] = amd_bytealign_S (in1[2], in1[1], 2); + out0[1] = amd_bytealign_S (in1[3], in1[2], 2); + out0[2] = amd_bytealign_S ( 0, in1[3], 2); + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 23: out0[0] = amd_bytealign_S (in1[2], in1[1], 3); + out0[1] = amd_bytealign_S (in1[3], in1[2], 3); + out0[2] = amd_bytealign_S ( 0, in1[3], 3); + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 24: out0[0] = in1[2]; + out0[1] = in1[3]; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 25: out0[0] = amd_bytealign_S (in1[3], in1[2], 1); + out0[1] = amd_bytealign_S ( 0, in1[3], 1); + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 26: out0[0] = amd_bytealign_S (in1[3], in1[2], 2); + out0[1] = amd_bytealign_S ( 0, in1[3], 2); + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 27: out0[0] = amd_bytealign_S (in1[3], in1[2], 3); + out0[1] = amd_bytealign_S ( 0, in1[3], 3); + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 28: out0[0] = in1[3]; + out0[1] = 0; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 29: out0[0] = amd_bytealign_S ( 0, in1[3], 1); + out0[1] = 0; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 30: out0[0] = amd_bytealign_S ( 0, in1[3], 2); + out0[1] = 0; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + case 31: out0[0] = amd_bytealign_S ( 0, in1[3], 3); + out0[1] = 0; + out0[2] = 0; + out0[3] = 0; + out1[0] = 0; + out1[1] = 0; + out1[2] = 0; + out1[3] = 0; + break; + } + #endif +} + +inline void rshift_block_N (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4], const u32 num) +{ + #ifdef IS_NV + switch (num) + { + case 0: out1[3] = in1[3]; + out1[2] = in1[2]; + out1[1] = in1[1]; + out1[0] = in1[0]; + out0[3] = in0[3]; + out0[2] = in0[2]; + out0[1] = in0[1]; + out0[0] = in0[0]; + break; + case 1: out1[3] = __byte_perm_S (in1[2], in1[3], 0x6543); + out1[2] = __byte_perm_S (in1[1], in1[2], 0x6543); + out1[1] = __byte_perm_S (in1[0], in1[1], 0x6543); + out1[0] = __byte_perm_S (in0[3], in1[0], 0x6543); + out0[3] = __byte_perm_S (in0[2], in0[3], 0x6543); + out0[2] = __byte_perm_S (in0[1], in0[2], 0x6543); + out0[1] = __byte_perm_S (in0[0], in0[1], 0x6543); + out0[0] = __byte_perm_S ( 0, in0[0], 0x6543); + break; + case 2: out1[3] = __byte_perm_S (in1[2], in1[3], 0x5432); + out1[2] = __byte_perm_S (in1[1], in1[2], 0x5432); + out1[1] = __byte_perm_S (in1[0], in1[1], 0x5432); + out1[0] = __byte_perm_S (in0[3], in1[0], 0x5432); + out0[3] = __byte_perm_S (in0[2], in0[3], 0x5432); + out0[2] = __byte_perm_S (in0[1], in0[2], 0x5432); + out0[1] = __byte_perm_S (in0[0], in0[1], 0x5432); + out0[0] = __byte_perm_S ( 0, in0[0], 0x5432); + break; + case 3: out1[3] = __byte_perm_S (in1[2], in1[3], 0x4321); + out1[2] = __byte_perm_S (in1[1], in1[2], 0x4321); + out1[1] = __byte_perm_S (in1[0], in1[1], 0x4321); + out1[0] = __byte_perm_S (in0[3], in1[0], 0x4321); + out0[3] = __byte_perm_S (in0[2], in0[3], 0x4321); + out0[2] = __byte_perm_S (in0[1], in0[2], 0x4321); + out0[1] = __byte_perm_S (in0[0], in0[1], 0x4321); + out0[0] = __byte_perm_S ( 0, in0[0], 0x4321); + break; + case 4: out1[3] = in1[2]; + out1[2] = in1[1]; + out1[1] = in1[0]; + out1[0] = in0[3]; + out0[3] = in0[2]; + out0[2] = in0[1]; + out0[1] = in0[0]; + out0[0] = 0; + break; + case 5: out1[3] = __byte_perm_S (in1[1], in1[2], 0x6543); + out1[2] = __byte_perm_S (in1[0], in1[1], 0x6543); + out1[1] = __byte_perm_S (in0[3], in1[0], 0x6543); + out1[0] = __byte_perm_S (in0[2], in0[3], 0x6543); + out0[3] = __byte_perm_S (in0[1], in0[2], 0x6543); + out0[2] = __byte_perm_S (in0[0], in0[1], 0x6543); + out0[1] = __byte_perm_S ( 0, in0[0], 0x6543); + out0[0] = 0; + break; + case 6: out1[3] = __byte_perm_S (in1[1], in1[2], 0x5432); + out1[2] = __byte_perm_S (in1[0], in1[1], 0x5432); + out1[1] = __byte_perm_S (in0[3], in1[0], 0x5432); + out1[0] = __byte_perm_S (in0[2], in0[3], 0x5432); + out0[3] = __byte_perm_S (in0[1], in0[2], 0x5432); + out0[2] = __byte_perm_S (in0[0], in0[1], 0x5432); + out0[1] = __byte_perm_S ( 0, in0[0], 0x5432); + out0[0] = 0; + break; + case 7: out1[3] = __byte_perm_S (in1[1], in1[2], 0x4321); + out1[2] = __byte_perm_S (in1[0], in1[1], 0x4321); + out1[1] = __byte_perm_S (in0[3], in1[0], 0x4321); + out1[0] = __byte_perm_S (in0[2], in0[3], 0x4321); + out0[3] = __byte_perm_S (in0[1], in0[2], 0x4321); + out0[2] = __byte_perm_S (in0[0], in0[1], 0x4321); + out0[1] = __byte_perm_S ( 0, in0[0], 0x4321); + out0[0] = 0; + break; + case 8: out1[3] = in1[1]; + out1[2] = in1[0]; + out1[1] = in0[3]; + out1[0] = in0[2]; + out0[3] = in0[1]; + out0[2] = in0[0]; + out0[1] = 0; + out0[0] = 0; + break; + case 9: out1[3] = __byte_perm_S (in1[0], in1[1], 0x6543); + out1[2] = __byte_perm_S (in0[3], in1[0], 0x6543); + out1[1] = __byte_perm_S (in0[2], in0[3], 0x6543); + out1[0] = __byte_perm_S (in0[1], in0[2], 0x6543); + out0[3] = __byte_perm_S (in0[0], in0[1], 0x6543); + out0[2] = __byte_perm_S ( 0, in0[0], 0x6543); + out0[1] = 0; + out0[0] = 0; + break; + case 10: out1[3] = __byte_perm_S (in1[0], in1[1], 0x5432); + out1[2] = __byte_perm_S (in0[3], in1[0], 0x5432); + out1[1] = __byte_perm_S (in0[2], in0[3], 0x5432); + out1[0] = __byte_perm_S (in0[1], in0[2], 0x5432); + out0[3] = __byte_perm_S (in0[0], in0[1], 0x5432); + out0[2] = __byte_perm_S ( 0, in0[0], 0x5432); + out0[1] = 0; + out0[0] = 0; + break; + case 11: out1[3] = __byte_perm_S (in1[0], in1[1], 0x4321); + out1[2] = __byte_perm_S (in0[3], in1[0], 0x4321); + out1[1] = __byte_perm_S (in0[2], in0[3], 0x4321); + out1[0] = __byte_perm_S (in0[1], in0[2], 0x4321); + out0[3] = __byte_perm_S (in0[0], in0[1], 0x4321); + out0[2] = __byte_perm_S ( 0, in0[0], 0x4321); + out0[1] = 0; + out0[0] = 0; + break; + case 12: out1[3] = in1[0]; + out1[2] = in0[3]; + out1[1] = in0[2]; + out1[0] = in0[1]; + out0[3] = in0[0]; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 13: out1[3] = __byte_perm_S (in0[3], in1[0], 0x6543); + out1[2] = __byte_perm_S (in0[2], in0[3], 0x6543); + out1[1] = __byte_perm_S (in0[1], in0[2], 0x6543); + out1[0] = __byte_perm_S (in0[0], in0[1], 0x6543); + out0[3] = __byte_perm_S ( 0, in0[0], 0x6543); + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 14: out1[3] = __byte_perm_S (in0[3], in1[0], 0x5432); + out1[2] = __byte_perm_S (in0[2], in0[3], 0x5432); + out1[1] = __byte_perm_S (in0[1], in0[2], 0x5432); + out1[0] = __byte_perm_S (in0[0], in0[1], 0x5432); + out0[3] = __byte_perm_S ( 0, in0[0], 0x5432); + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 15: out1[3] = __byte_perm_S (in0[3], in1[0], 0x4321); + out1[2] = __byte_perm_S (in0[2], in0[3], 0x4321); + out1[1] = __byte_perm_S (in0[1], in0[2], 0x4321); + out1[0] = __byte_perm_S (in0[0], in0[1], 0x4321); + out0[3] = __byte_perm_S ( 0, in0[0], 0x4321); + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 16: out1[3] = in0[3]; + out1[2] = in0[2]; + out1[1] = in0[1]; + out1[0] = in0[0]; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 17: out1[3] = __byte_perm_S (in0[2], in0[3], 0x6543); + out1[2] = __byte_perm_S (in0[1], in0[2], 0x6543); + out1[1] = __byte_perm_S (in0[0], in0[1], 0x6543); + out1[0] = __byte_perm_S ( 0, in0[0], 0x6543); + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 18: out1[3] = __byte_perm_S (in0[2], in0[3], 0x5432); + out1[2] = __byte_perm_S (in0[1], in0[2], 0x5432); + out1[1] = __byte_perm_S (in0[0], in0[1], 0x5432); + out1[0] = __byte_perm_S ( 0, in0[0], 0x5432); + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 19: out1[3] = __byte_perm_S (in0[2], in0[3], 0x4321); + out1[2] = __byte_perm_S (in0[1], in0[2], 0x4321); + out1[1] = __byte_perm_S (in0[0], in0[1], 0x4321); + out1[0] = __byte_perm_S ( 0, in0[0], 0x4321); + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 20: out1[3] = in0[2]; + out1[2] = in0[1]; + out1[1] = in0[0]; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 21: out1[3] = __byte_perm_S (in0[1], in0[2], 0x6543); + out1[2] = __byte_perm_S (in0[0], in0[1], 0x6543); + out1[1] = __byte_perm_S ( 0, in0[0], 0x6543); + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 22: out1[3] = __byte_perm_S (in0[1], in0[2], 0x5432); + out1[2] = __byte_perm_S (in0[0], in0[1], 0x5432); + out1[1] = __byte_perm_S ( 0, in0[0], 0x5432); + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 23: out1[3] = __byte_perm_S (in0[1], in0[2], 0x4321); + out1[2] = __byte_perm_S (in0[0], in0[1], 0x4321); + out1[1] = __byte_perm_S ( 0, in0[0], 0x4321); + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 24: out1[3] = in0[1]; + out1[2] = in0[0]; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 25: out1[3] = __byte_perm_S (in0[0], in0[1], 0x6543); + out1[2] = __byte_perm_S ( 0, in0[0], 0x6543); + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 26: out1[3] = __byte_perm_S (in0[0], in0[1], 0x5432); + out1[2] = __byte_perm_S ( 0, in0[0], 0x5432); + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 27: out1[3] = __byte_perm_S (in0[0], in0[1], 0x4321); + out1[2] = __byte_perm_S ( 0, in0[0], 0x4321); + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 28: out1[3] = in0[0]; + out1[2] = 0; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 29: out1[3] = __byte_perm_S ( 0, in0[0], 0x6543); + out1[2] = 0; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 30: out1[3] = __byte_perm_S ( 0, in0[0], 0x5432); + out1[2] = 0; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 31: out1[3] = __byte_perm_S ( 0, in0[0], 0x4321); + out1[2] = 0; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + } + #endif + + #if defined IS_AMD || defined IS_GENERIC + switch (num) + { + case 0: out1[3] = in1[3]; + out1[2] = in1[2]; + out1[1] = in1[1]; + out1[0] = in1[0]; + out0[3] = in0[3]; + out0[2] = in0[2]; + out0[1] = in0[1]; + out0[0] = in0[0]; + break; + case 1: out1[3] = amd_bytealign_S (in1[3], in1[2], 3); + out1[2] = amd_bytealign_S (in1[2], in1[1], 3); + out1[1] = amd_bytealign_S (in1[1], in1[0], 3); + out1[0] = amd_bytealign_S (in1[0], in0[3], 3); + out0[3] = amd_bytealign_S (in0[3], in0[2], 3); + out0[2] = amd_bytealign_S (in0[2], in0[1], 3); + out0[1] = amd_bytealign_S (in0[1], in0[0], 3); + out0[0] = amd_bytealign_S (in0[0], 0, 3); + break; + case 2: out1[3] = amd_bytealign_S (in1[3], in1[2], 2); + out1[2] = amd_bytealign_S (in1[2], in1[1], 2); + out1[1] = amd_bytealign_S (in1[1], in1[0], 2); + out1[0] = amd_bytealign_S (in1[0], in0[3], 2); + out0[3] = amd_bytealign_S (in0[3], in0[2], 2); + out0[2] = amd_bytealign_S (in0[2], in0[1], 2); + out0[1] = amd_bytealign_S (in0[1], in0[0], 2); + out0[0] = amd_bytealign_S (in0[0], 0, 2); + break; + case 3: out1[3] = amd_bytealign_S (in1[3], in1[2], 1); + out1[2] = amd_bytealign_S (in1[2], in1[1], 1); + out1[1] = amd_bytealign_S (in1[1], in1[0], 1); + out1[0] = amd_bytealign_S (in1[0], in0[3], 1); + out0[3] = amd_bytealign_S (in0[3], in0[2], 1); + out0[2] = amd_bytealign_S (in0[2], in0[1], 1); + out0[1] = amd_bytealign_S (in0[1], in0[0], 1); + out0[0] = amd_bytealign_S (in0[0], 0, 1); + break; + case 4: out1[3] = in1[2]; + out1[2] = in1[1]; + out1[1] = in1[0]; + out1[0] = in0[3]; + out0[3] = in0[2]; + out0[2] = in0[1]; + out0[1] = in0[0]; + out0[0] = 0; + break; + case 5: out1[3] = amd_bytealign_S (in1[2], in1[1], 3); + out1[2] = amd_bytealign_S (in1[1], in1[0], 3); + out1[1] = amd_bytealign_S (in1[0], in0[3], 3); + out1[0] = amd_bytealign_S (in0[3], in0[2], 3); + out0[3] = amd_bytealign_S (in0[2], in0[1], 3); + out0[2] = amd_bytealign_S (in0[1], in0[0], 3); + out0[1] = amd_bytealign_S (in0[0], 0, 3); + out0[0] = 0; + break; + case 6: out1[3] = amd_bytealign_S (in1[2], in1[1], 2); + out1[2] = amd_bytealign_S (in1[1], in1[0], 2); + out1[1] = amd_bytealign_S (in1[0], in0[3], 2); + out1[0] = amd_bytealign_S (in0[3], in0[2], 2); + out0[3] = amd_bytealign_S (in0[2], in0[1], 2); + out0[2] = amd_bytealign_S (in0[1], in0[0], 2); + out0[1] = amd_bytealign_S (in0[0], 0, 2); + out0[0] = 0; + break; + case 7: out1[3] = amd_bytealign_S (in1[2], in1[1], 1); + out1[2] = amd_bytealign_S (in1[1], in1[0], 1); + out1[1] = amd_bytealign_S (in1[0], in0[3], 1); + out1[0] = amd_bytealign_S (in0[3], in0[2], 1); + out0[3] = amd_bytealign_S (in0[2], in0[1], 1); + out0[2] = amd_bytealign_S (in0[1], in0[0], 1); + out0[1] = amd_bytealign_S (in0[0], 0, 1); + out0[0] = 0; + break; + case 8: out1[3] = in1[1]; + out1[2] = in1[0]; + out1[1] = in0[3]; + out1[0] = in0[2]; + out0[3] = in0[1]; + out0[2] = in0[0]; + out0[1] = 0; + out0[0] = 0; + break; + case 9: out1[3] = amd_bytealign_S (in1[1], in1[0], 3); + out1[2] = amd_bytealign_S (in1[0], in0[3], 3); + out1[1] = amd_bytealign_S (in0[3], in0[2], 3); + out1[0] = amd_bytealign_S (in0[2], in0[1], 3); + out0[3] = amd_bytealign_S (in0[1], in0[0], 3); + out0[2] = amd_bytealign_S (in0[0], 0, 3); + out0[1] = 0; + out0[0] = 0; + break; + case 10: out1[3] = amd_bytealign_S (in1[1], in1[0], 2); + out1[2] = amd_bytealign_S (in1[0], in0[3], 2); + out1[1] = amd_bytealign_S (in0[3], in0[2], 2); + out1[0] = amd_bytealign_S (in0[2], in0[1], 2); + out0[3] = amd_bytealign_S (in0[1], in0[0], 2); + out0[2] = amd_bytealign_S (in0[0], 0, 2); + out0[1] = 0; + out0[0] = 0; + break; + case 11: out1[3] = amd_bytealign_S (in1[1], in1[0], 1); + out1[2] = amd_bytealign_S (in1[0], in0[3], 1); + out1[1] = amd_bytealign_S (in0[3], in0[2], 1); + out1[0] = amd_bytealign_S (in0[2], in0[1], 1); + out0[3] = amd_bytealign_S (in0[1], in0[0], 1); + out0[2] = amd_bytealign_S (in0[0], 0, 1); + out0[1] = 0; + out0[0] = 0; + break; + case 12: out1[3] = in1[0]; + out1[2] = in0[3]; + out1[1] = in0[2]; + out1[0] = in0[1]; + out0[3] = in0[0]; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 13: out1[3] = amd_bytealign_S (in1[0], in0[3], 3); + out1[2] = amd_bytealign_S (in0[3], in0[2], 3); + out1[1] = amd_bytealign_S (in0[2], in0[1], 3); + out1[0] = amd_bytealign_S (in0[1], in0[0], 3); + out0[3] = amd_bytealign_S (in0[0], 0, 3); + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 14: out1[3] = amd_bytealign_S (in1[0], in0[3], 2); + out1[2] = amd_bytealign_S (in0[3], in0[2], 2); + out1[1] = amd_bytealign_S (in0[2], in0[1], 2); + out1[0] = amd_bytealign_S (in0[1], in0[0], 2); + out0[3] = amd_bytealign_S (in0[0], 0, 2); + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 15: out1[3] = amd_bytealign_S (in1[0], in0[3], 1); + out1[2] = amd_bytealign_S (in0[3], in0[2], 1); + out1[1] = amd_bytealign_S (in0[2], in0[1], 1); + out1[0] = amd_bytealign_S (in0[1], in0[0], 1); + out0[3] = amd_bytealign_S (in0[0], 0, 1); + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 16: out1[3] = in0[3]; + out1[2] = in0[2]; + out1[1] = in0[1]; + out1[0] = in0[0]; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 17: out1[3] = amd_bytealign_S (in0[3], in0[2], 3); + out1[2] = amd_bytealign_S (in0[2], in0[1], 3); + out1[1] = amd_bytealign_S (in0[1], in0[0], 3); + out1[0] = amd_bytealign_S (in0[0], 0, 3); + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 18: out1[3] = amd_bytealign_S (in0[3], in0[2], 2); + out1[2] = amd_bytealign_S (in0[2], in0[1], 2); + out1[1] = amd_bytealign_S (in0[1], in0[0], 2); + out1[0] = amd_bytealign_S (in0[0], 0, 2); + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 19: out1[3] = amd_bytealign_S (in0[3], in0[2], 1); + out1[2] = amd_bytealign_S (in0[2], in0[1], 1); + out1[1] = amd_bytealign_S (in0[1], in0[0], 1); + out1[0] = amd_bytealign_S (in0[0], 0, 1); + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 20: out1[3] = in0[2]; + out1[2] = in0[1]; + out1[1] = in0[0]; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 21: out1[3] = amd_bytealign_S (in0[2], in0[1], 3); + out1[2] = amd_bytealign_S (in0[1], in0[0], 3); + out1[1] = amd_bytealign_S (in0[0], 0, 3); + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 22: out1[3] = amd_bytealign_S (in0[2], in0[1], 2); + out1[2] = amd_bytealign_S (in0[1], in0[0], 2); + out1[1] = amd_bytealign_S (in0[0], 0, 2); + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 23: out1[3] = amd_bytealign_S (in0[2], in0[1], 1); + out1[2] = amd_bytealign_S (in0[1], in0[0], 1); + out1[1] = amd_bytealign_S (in0[0], 0, 1); + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 24: out1[3] = in0[1]; + out1[2] = in0[0]; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 25: out1[3] = amd_bytealign_S (in0[1], in0[0], 3); + out1[2] = amd_bytealign_S (in0[0], 0, 3); + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 26: out1[3] = amd_bytealign_S (in0[1], in0[0], 2); + out1[2] = amd_bytealign_S (in0[0], 0, 2); + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 27: out1[3] = amd_bytealign_S (in0[1], in0[0], 1); + out1[2] = amd_bytealign_S (in0[0], 0, 1); + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 28: out1[3] = in0[0]; + out1[2] = 0; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 29: out1[3] = amd_bytealign_S (in0[0], 0, 3); + out1[2] = 0; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 30: out1[3] = amd_bytealign_S (in0[0], 0, 2); + out1[2] = 0; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + case 31: out1[3] = amd_bytealign_S (in0[0], 0, 1); + out1[2] = 0; + out1[1] = 0; + out1[0] = 0; + out0[3] = 0; + out0[2] = 0; + out0[1] = 0; + out0[0] = 0; + break; + } + #endif +} + +inline void append_block1 (const u32 offset, u32 dst0[4], u32 dst1[4], const u32 src_r0) +{ + u32 tmp[2]; + + switch (offset & 3) + { + case 0: tmp[0] = src_r0; + tmp[1] = 0; + break; + case 1: tmp[0] = src_r0 << 8; + tmp[1] = src_r0 >> 24; + break; + case 2: tmp[0] = src_r0 << 16; + tmp[1] = src_r0 >> 16; + break; + case 3: tmp[0] = src_r0 << 24; + tmp[1] = src_r0 >> 8; + break; + } + + switch (offset / 4) + { + case 0: dst0[0] |= tmp[0]; + dst0[1] = tmp[1]; + break; + case 1: dst0[1] |= tmp[0]; + dst0[2] = tmp[1]; + break; + case 2: dst0[2] |= tmp[0]; + dst0[3] = tmp[1]; + break; + case 3: dst0[3] |= tmp[0]; + dst1[0] = tmp[1]; + break; + case 4: dst1[0] |= tmp[0]; + dst1[1] = tmp[1]; + break; + case 5: dst1[1] |= tmp[0]; + dst1[2] = tmp[1]; + break; + case 6: dst1[2] |= tmp[0]; + dst1[3] = tmp[1]; + break; + case 7: dst1[3] |= tmp[0]; + break; + } +} + +inline void append_block8 (const u32 offset, u32 dst0[4], u32 dst1[4], const u32 src_l0[4], const u32 src_l1[4], const u32 src_r0[4], const u32 src_r1[4]) +{ +/* + #ifdef IS_NV + switch (offset) + { + case 0: + dst0[0] = src_r0[0]; + dst0[1] = src_r0[1]; + dst0[2] = src_r0[2]; + dst0[3] = src_r0[3]; + dst1[0] = src_r1[0]; + dst1[1] = src_r1[1]; + dst1[2] = src_r1[2]; + dst1[3] = src_r1[3]; + break; + + case 1: + dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x6540); + dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); + dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); + dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); + dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543); + dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543); + dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x6543); + dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x6543); + break; + + case 2: + dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x5410); + dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); + dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); + dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); + dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432); + dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432); + dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x5432); + dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x5432); + break; + + case 3: + dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x4210); + dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); + dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); + dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); + dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321); + dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321); + dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x4321); + dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x4321); + break; + + case 4: + dst0[1] = src_r0[0]; + dst0[2] = src_r0[1]; + dst0[3] = src_r0[2]; + dst1[0] = src_r0[3]; + dst1[1] = src_r1[0]; + dst1[2] = src_r1[1]; + dst1[3] = src_r1[2]; + break; + + case 5: + dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x6540); + dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); + dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); + dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); + dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543); + dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543); + dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x6543); + break; + + case 6: + dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x5410); + dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); + dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); + dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); + dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432); + dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432); + dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x5432); + break; + + case 7: + dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x4210); + dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); + dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); + dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); + dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321); + dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321); + dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x4321); + break; + + case 8: + dst0[2] = src_r0[0]; + dst0[3] = src_r0[1]; + dst1[0] = src_r0[2]; + dst1[1] = src_r0[3]; + dst1[2] = src_r1[0]; + dst1[3] = src_r1[1]; + break; + + case 9: + dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x6540); + dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); + dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); + dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); + dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543); + dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543); + break; + + case 10: + dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x5410); + dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); + dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); + dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); + dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432); + dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432); + break; + + case 11: + dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x4210); + dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); + dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); + dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); + dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321); + dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321); + break; + + case 12: + dst0[3] = src_r0[0]; + dst1[0] = src_r0[1]; + dst1[1] = src_r0[2]; + dst1[2] = src_r0[3]; + dst1[3] = src_r1[0]; + break; + + case 13: + dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x6540); + dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); + dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); + dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); + dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543); + break; + + case 14: + dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x5410); + dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); + dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); + dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); + dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432); + break; + + case 15: + dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x4210); + dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); + dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); + dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); + dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321); + break; + + case 16: + dst1[0] = src_r0[0]; + dst1[1] = src_r0[1]; + dst1[2] = src_r0[2]; + dst1[3] = src_r0[3]; + break; + + case 17: + dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x6540); + dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); + dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); + dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); + break; + + case 18: + dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x5410); + dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); + dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); + dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); + break; + + case 19: + dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x4210); + dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); + dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); + dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); + break; + + case 20: + dst1[1] = src_r0[0]; + dst1[2] = src_r0[1]; + dst1[3] = src_r0[2]; + break; + + case 21: + dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x6540); + dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); + dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); + break; + + case 22: + dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x5410); + dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); + dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); + break; + + case 23: + dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x4210); + dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); + dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); + break; + + case 24: + dst1[2] = src_r0[0]; + dst1[3] = src_r0[1]; + break; + + case 25: + dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x6540); + dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); + break; + + case 26: + dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x5410); + dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); + break; + + case 27: + dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x4210); + dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); + break; + + case 28: + dst1[3] = src_r0[0]; + break; + + case 29: + dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x6540); + break; + + case 30: + dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x5410); + break; + + case 31: + dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x4210); + break; + } + #endif + + #if defined IS_AMD || defined IS_GENERIC +*/ + switch (offset) + { + case 31: + dst1[3] = src_l1[3] | src_r0[0] << 24; + break; + case 30: + dst1[3] = src_l1[3] | src_r0[0] << 16; + break; + case 29: + dst1[3] = src_l1[3] | src_r0[0] << 8; + break; + case 28: + dst1[3] = src_r0[0]; + break; + case 27: + dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 1); + dst1[2] = src_l1[2] | src_r0[0] << 24; + break; + case 26: + dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 2); + dst1[2] = src_l1[2] | src_r0[0] << 16; + break; + case 25: + dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 3); + dst1[2] = src_l1[2] | src_r0[0] << 8; + break; + case 24: + dst1[3] = src_r0[1]; + dst1[2] = src_r0[0]; + break; + case 23: + dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 1); + dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 1); + dst1[1] = src_l1[1] | src_r0[0] << 24; + break; + case 22: + dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 2); + dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 2); + dst1[1] = src_l1[1] | src_r0[0] << 16; + break; + case 21: + dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 3); + dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 3); + dst1[1] = src_l1[1] | src_r0[0] << 8; + break; + case 20: + dst1[3] = src_r0[2]; + dst1[2] = src_r0[1]; + dst1[1] = src_r0[0]; + break; + case 19: + dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 1); + dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 1); + dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 1); + dst1[0] = src_l1[0] | src_r0[0] << 24; + break; + case 18: + dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 2); + dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 2); + dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 2); + dst1[0] = src_l1[0] | src_r0[0] << 16; + break; + case 17: + dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 3); + dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 3); + dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 3); + dst1[0] = src_l1[0] | src_r0[0] << 8; + break; + case 16: + dst1[3] = src_r0[3]; + dst1[2] = src_r0[2]; + dst1[1] = src_r0[1]; + dst1[0] = src_r0[0]; + break; + case 15: + dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 1); + dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 1); + dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 1); + dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 1); + dst0[3] = src_l0[3] | src_r0[0] << 24; + break; + case 14: + dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 2); + dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 2); + dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 2); + dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 2); + dst0[3] = src_l0[3] | src_r0[0] << 16; + break; + case 13: + dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 3); + dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 3); + dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 3); + dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 3); + dst0[3] = src_l0[3] | src_r0[0] << 8; + break; + case 12: + dst1[3] = src_r1[0]; + dst1[2] = src_r0[3]; + dst1[1] = src_r0[2]; + dst1[0] = src_r0[1]; + dst0[3] = src_r0[0]; + break; + case 11: + dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 1); + dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 1); + dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 1); + dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 1); + dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 1); + dst0[2] = src_l0[2] | src_r0[0] << 24; + break; + case 10: + dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 2); + dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 2); + dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 2); + dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 2); + dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 2); + dst0[2] = src_l0[2] | src_r0[0] << 16; + break; + case 9: + dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 3); + dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 3); + dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 3); + dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 3); + dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 3); + dst0[2] = src_l0[2] | src_r0[0] << 8; + break; + case 8: + dst1[3] = src_r1[1]; + dst1[2] = src_r1[0]; + dst1[1] = src_r0[3]; + dst1[0] = src_r0[2]; + dst0[3] = src_r0[1]; + dst0[2] = src_r0[0]; + break; + case 7: + dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 1); + dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 1); + dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 1); + dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 1); + dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 1); + dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 1); + dst0[1] = src_l0[1] | src_r0[0] << 24; + break; + case 6: + dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 2); + dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 2); + dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 2); + dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 2); + dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 2); + dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 2); + dst0[1] = src_l0[1] | src_r0[0] << 16; + break; + case 5: + dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 3); + dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 3); + dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 3); + dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 3); + dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 3); + dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 3); + dst0[1] = src_l0[1] | src_r0[0] << 8; + break; + case 4: + dst1[3] = src_r1[2]; + dst1[2] = src_r1[1]; + dst1[1] = src_r1[0]; + dst1[0] = src_r0[3]; + dst0[3] = src_r0[2]; + dst0[2] = src_r0[1]; + dst0[1] = src_r0[0]; + break; + case 3: + dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 1); + dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 1); + dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 1); + dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 1); + dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 1); + dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 1); + dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 1); + dst0[0] = src_l0[0] | src_r0[0] << 24; + break; + case 2: + dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 2); + dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 2); + dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 2); + dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 2); + dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 2); + dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 2); + dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 2); + dst0[0] = src_l0[0] | src_r0[0] << 16; + break; + case 1: + dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 3); + dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 3); + dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 3); + dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 3); + dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 3); + dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 3); + dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 3); + dst0[0] = src_l0[0] | src_r0[0] << 8; + break; + case 0: + dst1[3] = src_r1[3]; + dst1[2] = src_r1[2]; + dst1[1] = src_r1[1]; + dst1[0] = src_r1[0]; + dst0[3] = src_r0[3]; + dst0[2] = src_r0[2]; + dst0[1] = src_r0[1]; + dst0[0] = src_r0[0]; + break; + } +// #endif +} + +inline void reverse_block (u32 in0[4], u32 in1[4], u32 out0[4], u32 out1[4], const u32 len) +{ + rshift_block_N (in0, in1, out0, out1, 32 - len); + + u32 tib40[4]; + u32 tib41[4]; + + tib40[0] = out1[3]; + tib40[1] = out1[2]; + tib40[2] = out1[1]; + tib40[3] = out1[0]; + tib41[0] = out0[3]; + tib41[1] = out0[2]; + tib41[2] = out0[1]; + tib41[3] = out0[0]; + + out0[0] = swap32_S (tib40[0]); + out0[1] = swap32_S (tib40[1]); + out0[2] = swap32_S (tib40[2]); + out0[3] = swap32_S (tib40[3]); + out1[0] = swap32_S (tib41[0]); + out1[1] = swap32_S (tib41[1]); + out1[2] = swap32_S (tib41[2]); + out1[3] = swap32_S (tib41[3]); +} + +inline u32 rule_op_mangle_lrest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + buf0[0] |= (generate_cmask (buf0[0])); + buf0[1] |= (generate_cmask (buf0[1])); + buf0[2] |= (generate_cmask (buf0[2])); + buf0[3] |= (generate_cmask (buf0[3])); + buf1[0] |= (generate_cmask (buf1[0])); + buf1[1] |= (generate_cmask (buf1[1])); + buf1[2] |= (generate_cmask (buf1[2])); + buf1[3] |= (generate_cmask (buf1[3])); + + return in_len; +} + +inline u32 rule_op_mangle_urest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + buf0[0] &= ~(generate_cmask (buf0[0])); + buf0[1] &= ~(generate_cmask (buf0[1])); + buf0[2] &= ~(generate_cmask (buf0[2])); + buf0[3] &= ~(generate_cmask (buf0[3])); + buf1[0] &= ~(generate_cmask (buf1[0])); + buf1[1] &= ~(generate_cmask (buf1[1])); + buf1[2] &= ~(generate_cmask (buf1[2])); + buf1[3] &= ~(generate_cmask (buf1[3])); + + return in_len; +} + +inline u32 rule_op_mangle_lrest_ufirst (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + rule_op_mangle_lrest (p0, p1, buf0, buf1, in_len); + + buf0[0] &= ~(0x00000020 & generate_cmask (buf0[0])); + + return in_len; +} + +inline u32 rule_op_mangle_urest_lfirst (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + rule_op_mangle_urest (p0, p1, buf0, buf1, in_len); + + buf0[0] |= (0x00000020 & generate_cmask (buf0[0])); + + return in_len; +} + +inline u32 rule_op_mangle_trest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + buf0[0] ^= (generate_cmask (buf0[0])); + buf0[1] ^= (generate_cmask (buf0[1])); + buf0[2] ^= (generate_cmask (buf0[2])); + buf0[3] ^= (generate_cmask (buf0[3])); + buf1[0] ^= (generate_cmask (buf1[0])); + buf1[1] ^= (generate_cmask (buf1[1])); + buf1[2] ^= (generate_cmask (buf1[2])); + buf1[3] ^= (generate_cmask (buf1[3])); + + return in_len; +} + +inline u32 rule_op_mangle_toggle_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + const u32 tmp = 0x20u << ((p0 & 3) * 8); + + switch (p0 / 4) + { + case 0: buf0[0] ^= (tmp & generate_cmask (buf0[0])); break; + case 1: buf0[1] ^= (tmp & generate_cmask (buf0[1])); break; + case 2: buf0[2] ^= (tmp & generate_cmask (buf0[2])); break; + case 3: buf0[3] ^= (tmp & generate_cmask (buf0[3])); break; + case 4: buf1[0] ^= (tmp & generate_cmask (buf1[0])); break; + case 5: buf1[1] ^= (tmp & generate_cmask (buf1[1])); break; + case 6: buf1[2] ^= (tmp & generate_cmask (buf1[2])); break; + case 7: buf1[3] ^= (tmp & generate_cmask (buf1[3])); break; + } + + return in_len; +} + +inline u32 rule_op_mangle_reverse (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + reverse_block (buf0, buf1, buf0, buf1, in_len); + + return in_len; +} + +inline u32 rule_op_mangle_dupeword (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if ((in_len + in_len) >= 32) return (in_len); + + u32 out_len = in_len; + + append_block8 (out_len, buf0, buf1, buf0, buf1, buf0, buf1); + + out_len += in_len; + + return out_len; +} + +inline u32 rule_op_mangle_dupeword_times (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (((in_len * p0) + in_len) >= 32) return (in_len); + + u32 out_len = in_len; + + u32 tib40[4]; + u32 tib41[4]; + + tib40[0] = buf0[0]; + tib40[1] = buf0[1]; + tib40[2] = buf0[2]; + tib40[3] = buf0[3]; + tib41[0] = buf1[0]; + tib41[1] = buf1[1]; + tib41[2] = buf1[2]; + tib41[3] = buf1[3]; + + for (u32 i = 0; i < p0; i++) + { + append_block8 (out_len, buf0, buf1, buf0, buf1, tib40, tib41); + + out_len += in_len; + } + + return out_len; +} + +inline u32 rule_op_mangle_reflect (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if ((in_len + in_len) >= 32) return (in_len); + + u32 out_len = in_len; + + u32 tib40[4]; + u32 tib41[4]; + + reverse_block (buf0, buf1, tib40, tib41, out_len); + + append_block8 (out_len, buf0, buf1, buf0, buf1, tib40, tib41); + + out_len += in_len; + + return out_len; +} + +inline u32 rule_op_mangle_append (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if ((in_len + 1) >= 32) return (in_len); + + u32 out_len = in_len; + + append_block1 (out_len, buf0, buf1, p0); + + out_len++; + + return out_len; +} + +inline u32 rule_op_mangle_prepend (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if ((in_len + 1) >= 32) return (in_len); + + u32 out_len = in_len; + + rshift_block (buf0, buf1, buf0, buf1); + + buf0[0] = buf0[0] | p0; + + out_len++; + + return out_len; +} + +inline u32 rule_op_mangle_rotate_left (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (in_len == 0) return (in_len); + + const u32 in_len1 = in_len - 1; + + const u32 sh = (in_len1 & 3) * 8; + + const u32 tmp = (buf0[0] & 0xff) << sh; + + lshift_block (buf0, buf1, buf0, buf1); + + switch (in_len1 / 4) + { + case 0: buf0[0] |= tmp; break; + case 1: buf0[1] |= tmp; break; + case 2: buf0[2] |= tmp; break; + case 3: buf0[3] |= tmp; break; + case 4: buf1[0] |= tmp; break; + case 5: buf1[1] |= tmp; break; + case 6: buf1[2] |= tmp; break; + case 7: buf1[3] |= tmp; break; + } + + return in_len; +} + +inline u32 rule_op_mangle_rotate_right (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (in_len == 0) return (in_len); + + const u32 in_len1 = in_len - 1; + + const u32 sh = (in_len1 & 3) * 8; + + u32 tmp = 0; + + switch (in_len1 / 4) + { + case 0: tmp = (buf0[0] >> sh) & 0xff; break; + case 1: tmp = (buf0[1] >> sh) & 0xff; break; + case 2: tmp = (buf0[2] >> sh) & 0xff; break; + case 3: tmp = (buf0[3] >> sh) & 0xff; break; + case 4: tmp = (buf1[0] >> sh) & 0xff; break; + case 5: tmp = (buf1[1] >> sh) & 0xff; break; + case 6: tmp = (buf1[2] >> sh) & 0xff; break; + case 7: tmp = (buf1[3] >> sh) & 0xff; break; + } + + rshift_block (buf0, buf1, buf0, buf1); + + buf0[0] |= tmp; + + truncate_right (buf0, buf1, in_len); + + return in_len; +} + +inline u32 rule_op_mangle_delete_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (in_len == 0) return (in_len); + + const u32 in_len1 = in_len - 1; + + lshift_block (buf0, buf1, buf0, buf1); + + return in_len1; +} + +inline u32 rule_op_mangle_delete_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (in_len == 0) return (in_len); + + const u32 in_len1 = in_len - 1; + + const u32 tmp = (1 << ((in_len1 & 3) * 8)) - 1; + + switch (in_len1 / 4) + { + case 0: buf0[0] &= tmp; break; + case 1: buf0[1] &= tmp; break; + case 2: buf0[2] &= tmp; break; + case 3: buf0[3] &= tmp; break; + case 4: buf1[0] &= tmp; break; + case 5: buf1[1] &= tmp; break; + case 6: buf1[2] &= tmp; break; + case 7: buf1[3] &= tmp; break; + } + + return in_len1; +} + +inline u32 rule_op_mangle_delete_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + u32 out_len = in_len; + + u32 tib40[4]; + u32 tib41[4]; + + lshift_block (buf0, buf1, tib40, tib41); + + const u32 ml = (1 << ((p0 & 3) * 8)) - 1; + const u32 mr = ~ml; + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & ml) + | (tib40[0] & mr); + buf0[1] = tib40[1]; + buf0[2] = tib40[2]; + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 1: buf0[1] = (buf0[1] & ml) + | (tib40[1] & mr); + buf0[2] = tib40[2]; + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 2: buf0[2] = (buf0[2] & ml) + | (tib40[2] & mr); + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 3: buf0[3] = (buf0[3] & ml) + | (tib40[3] & mr); + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 4: buf1[0] = (buf1[0] & ml) + | (tib41[0] & mr); + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 5: buf1[1] = (buf1[1] & ml) + | (tib41[1] & mr); + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 6: buf1[2] = (buf1[2] & ml) + | (tib41[2] & mr); + buf1[3] = tib41[3]; + break; + case 7: buf1[3] = (buf1[3] & ml) + | (tib41[3] & mr); + break; + } + + out_len--; + + return out_len; +} + +inline u32 rule_op_mangle_extract (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + if ((p0 + p1) > in_len) return (in_len); + + u32 out_len = p1; + + lshift_block_N (buf0, buf1, buf0, buf1, p0); + + truncate_right (buf0, buf1, out_len); + + return out_len; +} + +inline u32 rule_op_mangle_omit (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + if ((p0 + p1) > in_len) return (in_len); + + u32 out_len = in_len; + + u32 tib40[4]; + u32 tib41[4]; + + tib40[0] = 0; + tib40[1] = 0; + tib40[2] = 0; + tib40[3] = 0; + tib41[0] = 0; + tib41[1] = 0; + tib41[2] = 0; + tib41[3] = 0; + + lshift_block_N (buf0, buf1, tib40, tib41, p1); + + const u32 ml = (1 << ((p0 & 3) * 8)) - 1; + const u32 mr = ~ml; + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & ml) + | (tib40[0] & mr); + buf0[1] = tib40[1]; + buf0[2] = tib40[2]; + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 1: buf0[1] = (buf0[1] & ml) + | (tib40[1] & mr); + buf0[2] = tib40[2]; + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 2: buf0[2] = (buf0[2] & ml) + | (tib40[2] & mr); + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 3: buf0[3] = (buf0[3] & ml) + | (tib40[3] & mr); + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 4: buf1[0] = (buf1[0] & ml) + | (tib41[0] & mr); + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 5: buf1[1] = (buf1[1] & ml) + | (tib41[1] & mr); + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 6: buf1[2] = (buf1[2] & ml) + | (tib41[2] & mr); + buf1[3] = tib41[3]; + break; + case 7: buf1[3] = (buf1[3] & ml) + | (tib41[3] & mr); + break; + } + + out_len -= p1; + + return out_len; +} + +inline u32 rule_op_mangle_insert (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 > in_len) return (in_len); + + if ((in_len + 1) >= 32) return (in_len); + + u32 out_len = in_len; + + u32 tib40[4]; + u32 tib41[4]; + + rshift_block (buf0, buf1, tib40, tib41); + + const u32 p1n = p1 << ((p0 & 3) * 8); + + const u32 ml = (1 << ((p0 & 3) * 8)) - 1; + + const u32 mr = 0xffffff00 << ((p0 & 3) * 8); + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & ml) | p1n | (tib40[0] & mr); + buf0[1] = tib40[1]; + buf0[2] = tib40[2]; + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 1: buf0[1] = (buf0[1] & ml) | p1n | (tib40[1] & mr); + buf0[2] = tib40[2]; + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 2: buf0[2] = (buf0[2] & ml) | p1n | (tib40[2] & mr); + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 3: buf0[3] = (buf0[3] & ml) | p1n | (tib40[3] & mr); + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 4: buf1[0] = (buf1[0] & ml) | p1n | (tib41[0] & mr); + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 5: buf1[1] = (buf1[1] & ml) | p1n | (tib41[1] & mr); + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + break; + case 6: buf1[2] = (buf1[2] & ml) | p1n | (tib41[2] & mr); + buf1[3] = tib41[3]; + break; + case 7: buf1[3] = (buf1[3] & ml) | p1n | (tib41[3] & mr); + break; + } + + out_len++; + + return out_len; +} + +inline u32 rule_op_mangle_overstrike (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + const u32 p1n = p1 << ((p0 & 3) * 8); + + const u32 m = ~(0xffu << ((p0 & 3) * 8)); + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & m) | p1n; break; + case 1: buf0[1] = (buf0[1] & m) | p1n; break; + case 2: buf0[2] = (buf0[2] & m) | p1n; break; + case 3: buf0[3] = (buf0[3] & m) | p1n; break; + case 4: buf1[0] = (buf1[0] & m) | p1n; break; + case 5: buf1[1] = (buf1[1] & m) | p1n; break; + case 6: buf1[2] = (buf1[2] & m) | p1n; break; + case 7: buf1[3] = (buf1[3] & m) | p1n; break; + } + + return in_len; +} + +inline u32 rule_op_mangle_truncate_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + truncate_right (buf0, buf1, p0); + + return p0; +} + +inline u32 rule_op_mangle_replace (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + #ifdef IS_NV + for (u32 i = 0; i < in_len; i++) + { + switch (i) + { + case 0: if ((__byte_perm_S (buf0[0], 0, 0x6540)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7650); + break; + case 1: if ((__byte_perm_S (buf0[0], 0, 0x6541)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7604); + break; + case 2: if ((__byte_perm_S (buf0[0], 0, 0x6542)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7054); + break; + case 3: if ((__byte_perm_S (buf0[0], 0, 0x6543)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x0654); + break; + case 4: if ((__byte_perm_S (buf0[1], 0, 0x6540)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7650); + break; + case 5: if ((__byte_perm_S (buf0[1], 0, 0x6541)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7604); + break; + case 6: if ((__byte_perm_S (buf0[1], 0, 0x6542)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7054); + break; + case 7: if ((__byte_perm_S (buf0[1], 0, 0x6543)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x0654); + break; + case 8: if ((__byte_perm_S (buf0[2], 0, 0x6540)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7650); + break; + case 9: if ((__byte_perm_S (buf0[2], 0, 0x6541)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7604); + break; + case 10: if ((__byte_perm_S (buf0[2], 0, 0x6542)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7054); + break; + case 11: if ((__byte_perm_S (buf0[2], 0, 0x6543)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x0654); + break; + case 12: if ((__byte_perm_S (buf0[3], 0, 0x6540)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7650); + break; + case 13: if ((__byte_perm_S (buf0[3], 0, 0x6541)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7604); + break; + case 14: if ((__byte_perm_S (buf0[3], 0, 0x6542)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7054); + break; + case 15: if ((__byte_perm_S (buf0[3], 0, 0x6543)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x0654); + break; + case 16: if ((__byte_perm_S (buf1[0], 0, 0x6540)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7650); + break; + case 17: if ((__byte_perm_S (buf1[0], 0, 0x6541)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7604); + break; + case 18: if ((__byte_perm_S (buf1[0], 0, 0x6542)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7054); + break; + case 19: if ((__byte_perm_S (buf1[0], 0, 0x6543)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x0654); + break; + case 20: if ((__byte_perm_S (buf1[1], 0, 0x6540)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7650); + break; + case 21: if ((__byte_perm_S (buf1[1], 0, 0x6541)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7604); + break; + case 22: if ((__byte_perm_S (buf1[1], 0, 0x6542)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7054); + break; + case 23: if ((__byte_perm_S (buf1[1], 0, 0x6543)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x0654); + break; + case 24: if ((__byte_perm_S (buf1[2], 0, 0x6540)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7650); + break; + case 25: if ((__byte_perm_S (buf1[2], 0, 0x6541)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7604); + break; + case 26: if ((__byte_perm_S (buf1[2], 0, 0x6542)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7054); + break; + case 27: if ((__byte_perm_S (buf1[2], 0, 0x6543)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x0654); + break; + case 28: if ((__byte_perm_S (buf1[3], 0, 0x6540)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7650); + break; + case 29: if ((__byte_perm_S (buf1[3], 0, 0x6541)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7604); + break; + case 30: if ((__byte_perm_S (buf1[3], 0, 0x6542)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7054); + break; + case 31: if ((__byte_perm_S (buf1[3], 0, 0x6543)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x0654); + break; + } + } + #endif + + #if defined IS_AMD || defined IS_GENERIC + const uchar4 tmp0 = (uchar4) (p0); + const uchar4 tmp1 = (uchar4) (p1); + + uchar4 tmp; + + tmp = as_uchar4 (buf0[0]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[0] = as_uint (tmp); + tmp = as_uchar4 (buf0[1]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[1] = as_uint (tmp); + tmp = as_uchar4 (buf0[2]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[2] = as_uint (tmp); + tmp = as_uchar4 (buf0[3]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[3] = as_uint (tmp); + tmp = as_uchar4 (buf1[0]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[0] = as_uint (tmp); + tmp = as_uchar4 (buf1[1]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[1] = as_uint (tmp); + tmp = as_uchar4 (buf1[2]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[2] = as_uint (tmp); + tmp = as_uchar4 (buf1[3]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[3] = as_uint (tmp); + #endif + + return in_len; +} + +inline u32 rule_op_mangle_purgechar (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + // TODO + return in_len; +} + +inline u32 rule_op_mangle_togglecase_rec (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + // TODO + return in_len; +} + +inline u32 rule_op_mangle_dupechar_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if ( in_len == 0) return (in_len); + if ((in_len + p0) >= 32) return (in_len); + + u32 out_len = in_len; + + const u32 tmp = buf0[0] & 0xFF; + + rshift_block_N (buf0, buf1, buf0, buf1, p0); + + #ifdef IS_NV + switch (p0) + { + case 1: buf0[0] |= tmp; + break; + case 2: buf0[0] |= __byte_perm_S (tmp, 0, 0x5400); + break; + case 3: buf0[0] |= __byte_perm_S (tmp, 0, 0x4000); + break; + case 4: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + break; + case 5: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= tmp; + break; + case 6: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x5400); + break; + case 7: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x4000); + break; + case 8: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + break; + case 9: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= tmp; + break; + case 10: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x5400); + break; + case 11: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x4000); + break; + case 12: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + break; + case 13: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= tmp; + break; + case 14: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x5400); + break; + case 15: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x4000); + break; + case 16: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + break; + case 17: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= tmp; + break; + case 18: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x5400); + break; + case 19: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x4000); + break; + case 20: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + break; + case 21: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= tmp; + break; + case 22: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x5400); + break; + case 23: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x4000); + break; + case 24: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); + break; + case 25: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[2] |= tmp; + break; + case 26: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[2] |= __byte_perm_S (tmp, 0, 0x5400); + break; + case 27: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[2] |= __byte_perm_S (tmp, 0, 0x4000); + break; + case 28: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[2] |= __byte_perm_S (tmp, 0, 0x0000); + break; + case 29: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[3] |= tmp; + break; + case 30: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[3] |= __byte_perm_S (tmp, 0, 0x5400); + break; + case 31: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[2] |= __byte_perm_S (tmp, 0, 0x0000); + buf1[3] |= __byte_perm_S (tmp, 0, 0x4000); + break; + } + #endif + + #if defined IS_AMD || defined IS_GENERIC + switch (p0) + { + case 1: buf0[0] |= tmp << 0; + break; + case 2: buf0[0] |= tmp << 0 | tmp << 8; + break; + case 3: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16; + break; + case 4: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + break; + case 5: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0; + break; + case 6: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8; + break; + case 7: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16; + break; + case 8: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + break; + case 9: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0; + break; + case 10: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8; + break; + case 11: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16; + break; + case 12: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + break; + case 13: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0; + break; + case 14: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8; + break; + case 15: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16; + break; + case 16: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + break; + case 17: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0; + break; + case 18: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8; + break; + case 19: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16; + break; + case 20: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + break; + case 21: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0; + break; + case 22: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8; + break; + case 23: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16; + break; + case 24: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + break; + case 25: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[2] |= tmp << 0; + break; + case 26: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[2] |= tmp << 0 | tmp << 8; + break; + case 27: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16; + break; + case 28: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + break; + case 29: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[3] |= tmp << 0; + break; + case 30: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[3] |= tmp << 0 | tmp << 8; + break; + case 31: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; + buf1[3] |= tmp << 0 | tmp << 8 | tmp << 16; + break; + } + #endif + + out_len += p0; + + return out_len; +} + +inline u32 rule_op_mangle_dupechar_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if ( in_len == 0) return (in_len); + if ((in_len + p0) >= 32) return (in_len); + + const u32 in_len1 = in_len - 1; + + const u32 sh = (in_len1 & 3) * 8; + + u32 tmp = 0; + + switch (in_len1 / 4) + { + case 0: tmp = (buf0[0] >> sh) & 0xff; break; + case 1: tmp = (buf0[1] >> sh) & 0xff; break; + case 2: tmp = (buf0[2] >> sh) & 0xff; break; + case 3: tmp = (buf0[3] >> sh) & 0xff; break; + case 4: tmp = (buf1[0] >> sh) & 0xff; break; + case 5: tmp = (buf1[1] >> sh) & 0xff; break; + case 6: tmp = (buf1[2] >> sh) & 0xff; break; + case 7: tmp = (buf1[3] >> sh) & 0xff; break; + } + + u32 out_len = in_len; + + for (u32 i = 0; i < p0; i++) + { + append_block1 (out_len, buf0, buf1, tmp); + + out_len++; + } + + return out_len; +} + +inline u32 rule_op_mangle_dupechar_all (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if ( in_len == 0) return (in_len); + if ((in_len + in_len) >= 32) return (in_len); + + u32 out_len = in_len; + + u32 tib40[4]; + u32 tib41[4]; + + #ifdef IS_NV + tib40[0] = __byte_perm_S (buf0[0], 0, 0x1100); + tib40[1] = __byte_perm_S (buf0[0], 0, 0x3322); + tib40[2] = __byte_perm_S (buf0[1], 0, 0x1100); + tib40[3] = __byte_perm_S (buf0[1], 0, 0x3322); + tib41[0] = __byte_perm_S (buf0[2], 0, 0x1100); + tib41[1] = __byte_perm_S (buf0[2], 0, 0x3322); + tib41[2] = __byte_perm_S (buf0[3], 0, 0x1100); + tib41[3] = __byte_perm_S (buf0[3], 0, 0x3322); + + buf0[0] = tib40[0]; + buf0[1] = tib40[1]; + buf0[2] = tib40[2]; + buf0[3] = tib40[3]; + buf1[0] = tib41[0]; + buf1[1] = tib41[1]; + buf1[2] = tib41[2]; + buf1[3] = tib41[3]; + #endif + + #if defined IS_AMD || defined IS_GENERIC + tib40[0] = ((buf0[0] & 0x000000FF) << 0) | ((buf0[0] & 0x0000FF00) << 8); + tib40[1] = ((buf0[0] & 0x00FF0000) >> 16) | ((buf0[0] & 0xFF000000) >> 8); + tib40[2] = ((buf0[1] & 0x000000FF) << 0) | ((buf0[1] & 0x0000FF00) << 8); + tib40[3] = ((buf0[1] & 0x00FF0000) >> 16) | ((buf0[1] & 0xFF000000) >> 8); + tib41[0] = ((buf0[2] & 0x000000FF) << 0) | ((buf0[2] & 0x0000FF00) << 8); + tib41[1] = ((buf0[2] & 0x00FF0000) >> 16) | ((buf0[2] & 0xFF000000) >> 8); + tib41[2] = ((buf0[3] & 0x000000FF) << 0) | ((buf0[3] & 0x0000FF00) << 8); + tib41[3] = ((buf0[3] & 0x00FF0000) >> 16) | ((buf0[3] & 0xFF000000) >> 8); + + buf0[0] = tib40[0] | (tib40[0] << 8); + buf0[1] = tib40[1] | (tib40[1] << 8); + buf0[2] = tib40[2] | (tib40[2] << 8); + buf0[3] = tib40[3] | (tib40[3] << 8); + buf1[0] = tib41[0] | (tib41[0] << 8); + buf1[1] = tib41[1] | (tib41[1] << 8); + buf1[2] = tib41[2] | (tib41[2] << 8); + buf1[3] = tib41[3] | (tib41[3] << 8); + #endif + + out_len = out_len + out_len; + + return out_len; +} + +inline u32 rule_op_mangle_switch_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (in_len < 2) return (in_len); + + #ifdef IS_NV + buf0[0] = __byte_perm_S (buf0[0], 0, 0x3201); + #endif + + #if defined IS_AMD || defined IS_GENERIC + buf0[0] = (buf0[0] & 0xFFFF0000) | ((buf0[0] << 8) & 0x0000FF00) | ((buf0[0] >> 8) & 0x000000FF); + #endif + + return in_len; +} + +inline u32 rule_op_mangle_switch_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (in_len < 2) return (in_len); + + #ifdef IS_NV + switch (in_len) + { + case 2: buf0[0] = __byte_perm_S (buf0[0], 0, 0x5401); + break; + case 3: buf0[0] = __byte_perm_S (buf0[0], 0, 0x4120); + break; + case 4: buf0[0] = __byte_perm_S (buf0[0], 0, 0x2310); + break; + case 5: buf0[1] = __byte_perm_S (buf0[1], buf0[0], 0x7210); + buf0[0] = __byte_perm_S (buf0[0], buf0[1], 0x4210); + buf0[1] = __byte_perm_S (buf0[1], 0, 0x6543); + break; + case 6: buf0[1] = __byte_perm_S (buf0[1], 0, 0x5401); + break; + case 7: buf0[1] = __byte_perm_S (buf0[1], 0, 0x4120); + break; + case 8: buf0[1] = __byte_perm_S (buf0[1], 0, 0x2310); + break; + case 9: buf0[2] = __byte_perm_S (buf0[2], buf0[1], 0x7210); + buf0[1] = __byte_perm_S (buf0[1], buf0[2], 0x4210); + buf0[2] = __byte_perm_S (buf0[2], 0, 0x6543); + break; + case 10: buf0[2] = __byte_perm_S (buf0[2], 0, 0x5401); + break; + case 11: buf0[2] = __byte_perm_S (buf0[2], 0, 0x4120); + break; + case 12: buf0[2] = __byte_perm_S (buf0[2], 0, 0x2310); + break; + case 13: buf0[3] = __byte_perm_S (buf0[3], buf0[2], 0x7210); + buf0[2] = __byte_perm_S (buf0[2], buf0[3], 0x4210); + buf0[3] = __byte_perm_S (buf0[3], 0, 0x6543); + break; + case 14: buf0[3] = __byte_perm_S (buf0[3], 0, 0x5401); + break; + case 15: buf0[3] = __byte_perm_S (buf0[3], 0, 0x4120); + break; + case 16: buf0[3] = __byte_perm_S (buf0[3], 0, 0x2310); + break; + case 17: buf1[0] = __byte_perm_S (buf1[0], buf0[3], 0x7210); + buf0[3] = __byte_perm_S (buf0[3], buf1[0], 0x4210); + buf1[0] = __byte_perm_S (buf1[0], 0, 0x6543); + break; + case 18: buf1[0] = __byte_perm_S (buf1[0], 0, 0x5401); + break; + case 19: buf1[0] = __byte_perm_S (buf1[0], 0, 0x4120); + break; + case 20: buf1[0] = __byte_perm_S (buf1[0], 0, 0x2310); + break; + case 21: buf1[1] = __byte_perm_S (buf1[1], buf1[0], 0x7210); + buf1[0] = __byte_perm_S (buf1[0], buf1[1], 0x4210); + buf1[1] = __byte_perm_S (buf1[1], 0, 0x6543); + break; + case 22: buf1[1] = __byte_perm_S (buf1[1], 0, 0x5401); + break; + case 23: buf1[1] = __byte_perm_S (buf1[1], 0, 0x4120); + break; + case 24: buf1[1] = __byte_perm_S (buf1[1], 0, 0x2310); + break; + case 25: buf1[2] = __byte_perm_S (buf1[2], buf1[1], 0x7210); + buf1[1] = __byte_perm_S (buf1[1], buf1[2], 0x4210); + buf1[2] = __byte_perm_S (buf1[2], 0, 0x6543); + break; + case 26: buf1[2] = __byte_perm_S (buf1[2], 0, 0x5401); + break; + case 27: buf1[2] = __byte_perm_S (buf1[2], 0, 0x4120); + break; + case 28: buf1[2] = __byte_perm_S (buf1[2], 0, 0x2310); + break; + case 29: buf1[3] = __byte_perm_S (buf1[3], buf1[2], 0x7210); + buf1[2] = __byte_perm_S (buf1[2], buf1[3], 0x4210); + buf1[3] = __byte_perm_S (buf1[3], 0, 0x6543); + break; + case 30: buf1[3] = __byte_perm_S (buf1[3], 0, 0x5401); + break; + case 31: buf1[3] = __byte_perm_S (buf1[3], 0, 0x4120); + break; + } + #endif + + #if defined IS_AMD || defined IS_GENERIC + switch (in_len) + { + case 2: buf0[0] = ((buf0[0] << 8) & 0x0000FF00) | ((buf0[0] >> 8) & 0x000000FF); + break; + case 3: buf0[0] = (buf0[0] & 0x000000FF) | ((buf0[0] << 8) & 0x00FF0000) | ((buf0[0] >> 8) & 0x0000FF00); + break; + case 4: buf0[0] = (buf0[0] & 0x0000FFFF) | ((buf0[0] << 8) & 0xFF000000) | ((buf0[0] >> 8) & 0x00FF0000); + break; + case 5: buf0[1] = (buf0[0] & 0xFF000000) | buf0[1]; + buf0[0] = (buf0[0] & 0x00FFFFFF) | (buf0[1] << 24); + buf0[1] = (buf0[1] >> 24); + break; + case 6: buf0[1] = ((buf0[1] << 8) & 0x0000FF00) | ((buf0[1] >> 8) & 0x000000FF); + break; + case 7: buf0[1] = (buf0[1] & 0x000000FF) | ((buf0[1] << 8) & 0x00FF0000) | ((buf0[1] >> 8) & 0x0000FF00); + break; + case 8: buf0[1] = (buf0[1] & 0x0000FFFF) | ((buf0[1] << 8) & 0xFF000000) | ((buf0[1] >> 8) & 0x00FF0000); + break; + case 9: buf0[2] = (buf0[1] & 0xFF000000) | buf0[2]; + buf0[1] = (buf0[1] & 0x00FFFFFF) | (buf0[2] << 24); + buf0[2] = (buf0[2] >> 24); + break; + case 10: buf0[2] = ((buf0[2] << 8) & 0x0000FF00) | ((buf0[2] >> 8) & 0x000000FF); + break; + case 11: buf0[2] = (buf0[2] & 0x000000FF) | ((buf0[2] << 8) & 0x00FF0000) | ((buf0[2] >> 8) & 0x0000FF00); + break; + case 12: buf0[2] = (buf0[2] & 0x0000FFFF) | ((buf0[2] << 8) & 0xFF000000) | ((buf0[2] >> 8) & 0x00FF0000); + break; + case 13: buf0[3] = (buf0[2] & 0xFF000000) | buf0[3]; + buf0[2] = (buf0[2] & 0x00FFFFFF) | (buf0[3] << 24); + buf0[3] = (buf0[3] >> 24); + break; + case 14: buf0[3] = ((buf0[3] << 8) & 0x0000FF00) | ((buf0[3] >> 8) & 0x000000FF); + break; + case 15: buf0[3] = (buf0[3] & 0x000000FF) | ((buf0[3] << 8) & 0x00FF0000) | ((buf0[3] >> 8) & 0x0000FF00); + break; + case 16: buf0[3] = (buf0[3] & 0x0000FFFF) | ((buf0[3] << 8) & 0xFF000000) | ((buf0[3] >> 8) & 0x00FF0000); + break; + case 17: buf1[0] = (buf0[3] & 0xFF000000) | buf1[0]; + buf0[3] = (buf0[3] & 0x00FFFFFF) | (buf1[0] << 24); + buf1[0] = (buf1[0] >> 24); + break; + case 18: buf1[0] = ((buf1[0] << 8) & 0x0000FF00) | ((buf1[0] >> 8) & 0x000000FF); + break; + case 19: buf1[0] = (buf1[0] & 0x000000FF) | ((buf1[0] << 8) & 0x00FF0000) | ((buf1[0] >> 8) & 0x0000FF00); + break; + case 20: buf1[0] = (buf1[0] & 0x0000FFFF) | ((buf1[0] << 8) & 0xFF000000) | ((buf1[0] >> 8) & 0x00FF0000); + break; + case 21: buf1[1] = (buf1[0] & 0xFF000000) | buf1[1]; + buf1[0] = (buf1[0] & 0x00FFFFFF) | (buf1[1] << 24); + buf1[1] = (buf1[1] >> 24); + break; + case 22: buf1[1] = ((buf1[1] << 8) & 0x0000FF00) | ((buf1[1] >> 8) & 0x000000FF); + break; + case 23: buf1[1] = (buf1[1] & 0x000000FF) | ((buf1[1] << 8) & 0x00FF0000) | ((buf1[1] >> 8) & 0x0000FF00); + break; + case 24: buf1[1] = (buf1[1] & 0x0000FFFF) | ((buf1[1] << 8) & 0xFF000000) | ((buf1[1] >> 8) & 0x00FF0000); + break; + case 25: buf1[2] = (buf1[1] & 0xFF000000) | buf1[2]; + buf1[1] = (buf1[1] & 0x00FFFFFF) | (buf1[2] << 24); + buf1[2] = (buf1[2] >> 24); + break; + case 26: buf1[2] = ((buf1[2] << 8) & 0x0000FF00) | ((buf1[2] >> 8) & 0x000000FF); + break; + case 27: buf1[2] = (buf1[2] & 0x000000FF) | ((buf1[2] << 8) & 0x00FF0000) | ((buf1[2] >> 8) & 0x0000FF00); + break; + case 28: buf1[2] = (buf1[2] & 0x0000FFFF) | ((buf1[2] << 8) & 0xFF000000) | ((buf1[2] >> 8) & 0x00FF0000); + break; + case 29: buf1[3] = (buf1[2] & 0xFF000000) | buf1[3]; + buf1[2] = (buf1[2] & 0x00FFFFFF) | (buf1[3] << 24); + buf1[3] = (buf1[3] >> 24); + break; + case 30: buf1[3] = ((buf1[3] << 8) & 0x0000FF00) | ((buf1[3] >> 8) & 0x000000FF); + break; + case 31: buf1[3] = (buf1[3] & 0x000000FF) | ((buf1[3] << 8) & 0x00FF0000) | ((buf1[3] >> 8) & 0x0000FF00); + break; + } + #endif + + return in_len; +} + +inline u32 rule_op_mangle_switch_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + if (p1 >= in_len) return (in_len); + + u32 tmp0 = 0; + u32 tmp1 = 0; + + #ifdef IS_NV + switch (p0) + { + case 0: tmp0 = __byte_perm_S (buf0[0], 0, 0x6540); + break; + case 1: tmp0 = __byte_perm_S (buf0[0], 0, 0x6541); + break; + case 2: tmp0 = __byte_perm_S (buf0[0], 0, 0x6542); + break; + case 3: tmp0 = __byte_perm_S (buf0[0], 0, 0x6543); + break; + case 4: tmp0 = __byte_perm_S (buf0[1], 0, 0x6540); + break; + case 5: tmp0 = __byte_perm_S (buf0[1], 0, 0x6541); + break; + case 6: tmp0 = __byte_perm_S (buf0[1], 0, 0x6542); + break; + case 7: tmp0 = __byte_perm_S (buf0[1], 0, 0x6543); + break; + case 8: tmp0 = __byte_perm_S (buf0[2], 0, 0x6540); + break; + case 9: tmp0 = __byte_perm_S (buf0[2], 0, 0x6541); + break; + case 10: tmp0 = __byte_perm_S (buf0[2], 0, 0x6542); + break; + case 11: tmp0 = __byte_perm_S (buf0[2], 0, 0x6543); + break; + case 12: tmp0 = __byte_perm_S (buf0[3], 0, 0x6540); + break; + case 13: tmp0 = __byte_perm_S (buf0[3], 0, 0x6541); + break; + case 14: tmp0 = __byte_perm_S (buf0[3], 0, 0x6542); + break; + case 15: tmp0 = __byte_perm_S (buf0[3], 0, 0x6543); + break; + case 16: tmp0 = __byte_perm_S (buf1[0], 0, 0x6540); + break; + case 17: tmp0 = __byte_perm_S (buf1[0], 0, 0x6541); + break; + case 18: tmp0 = __byte_perm_S (buf1[0], 0, 0x6542); + break; + case 19: tmp0 = __byte_perm_S (buf1[0], 0, 0x6543); + break; + case 20: tmp0 = __byte_perm_S (buf1[1], 0, 0x6540); + break; + case 21: tmp0 = __byte_perm_S (buf1[1], 0, 0x6541); + break; + case 22: tmp0 = __byte_perm_S (buf1[1], 0, 0x6542); + break; + case 23: tmp0 = __byte_perm_S (buf1[1], 0, 0x6543); + break; + case 24: tmp0 = __byte_perm_S (buf1[2], 0, 0x6540); + break; + case 25: tmp0 = __byte_perm_S (buf1[2], 0, 0x6541); + break; + case 26: tmp0 = __byte_perm_S (buf1[2], 0, 0x6542); + break; + case 27: tmp0 = __byte_perm_S (buf1[2], 0, 0x6543); + break; + case 28: tmp0 = __byte_perm_S (buf1[3], 0, 0x6540); + break; + case 29: tmp0 = __byte_perm_S (buf1[3], 0, 0x6541); + break; + case 30: tmp0 = __byte_perm_S (buf1[3], 0, 0x6542); + break; + case 31: tmp0 = __byte_perm_S (buf1[3], 0, 0x6543); + break; + } + + switch (p1) + { + case 0: tmp1 = __byte_perm_S (buf0[0], 0, 0x6540); + buf0[0] = __byte_perm_S (tmp0, buf0[0], 0x7650); + break; + case 1: tmp1 = __byte_perm_S (buf0[0], 0, 0x6541); + buf0[0] = __byte_perm_S (tmp0, buf0[0], 0x7604); + break; + case 2: tmp1 = __byte_perm_S (buf0[0], 0, 0x6542); + buf0[0] = __byte_perm_S (tmp0, buf0[0], 0x7054); + break; + case 3: tmp1 = __byte_perm_S (buf0[0], 0, 0x6543); + buf0[0] = __byte_perm_S (tmp0, buf0[0], 0x0654); + break; + case 4: tmp1 = __byte_perm_S (buf0[1], 0, 0x6540); + buf0[1] = __byte_perm_S (tmp0, buf0[1], 0x7650); + break; + case 5: tmp1 = __byte_perm_S (buf0[1], 0, 0x6541); + buf0[1] = __byte_perm_S (tmp0, buf0[1], 0x7604); + break; + case 6: tmp1 = __byte_perm_S (buf0[1], 0, 0x6542); + buf0[1] = __byte_perm_S (tmp0, buf0[1], 0x7054); + break; + case 7: tmp1 = __byte_perm_S (buf0[1], 0, 0x6543); + buf0[1] = __byte_perm_S (tmp0, buf0[1], 0x0654); + break; + case 8: tmp1 = __byte_perm_S (buf0[2], 0, 0x6540); + buf0[2] = __byte_perm_S (tmp0, buf0[2], 0x7650); + break; + case 9: tmp1 = __byte_perm_S (buf0[2], 0, 0x6541); + buf0[2] = __byte_perm_S (tmp0, buf0[2], 0x7604); + break; + case 10: tmp1 = __byte_perm_S (buf0[2], 0, 0x6542); + buf0[2] = __byte_perm_S (tmp0, buf0[2], 0x7054); + break; + case 11: tmp1 = __byte_perm_S (buf0[2], 0, 0x6543); + buf0[2] = __byte_perm_S (tmp0, buf0[2], 0x0654); + break; + case 12: tmp1 = __byte_perm_S (buf0[3], 0, 0x6540); + buf0[3] = __byte_perm_S (tmp0, buf0[3], 0x7650); + break; + case 13: tmp1 = __byte_perm_S (buf0[3], 0, 0x6541); + buf0[3] = __byte_perm_S (tmp0, buf0[3], 0x7604); + break; + case 14: tmp1 = __byte_perm_S (buf0[3], 0, 0x6542); + buf0[3] = __byte_perm_S (tmp0, buf0[3], 0x7054); + break; + case 15: tmp1 = __byte_perm_S (buf0[3], 0, 0x6543); + buf0[3] = __byte_perm_S (tmp0, buf0[3], 0x0654); + break; + case 16: tmp1 = __byte_perm_S (buf1[0], 0, 0x6540); + buf1[0] = __byte_perm_S (tmp0, buf1[0], 0x7650); + break; + case 17: tmp1 = __byte_perm_S (buf1[0], 0, 0x6541); + buf1[0] = __byte_perm_S (tmp0, buf1[0], 0x7604); + break; + case 18: tmp1 = __byte_perm_S (buf1[0], 0, 0x6542); + buf1[0] = __byte_perm_S (tmp0, buf1[0], 0x7054); + break; + case 19: tmp1 = __byte_perm_S (buf1[0], 0, 0x6543); + buf1[0] = __byte_perm_S (tmp0, buf1[0], 0x0654); + break; + case 20: tmp1 = __byte_perm_S (buf1[1], 0, 0x6540); + buf1[1] = __byte_perm_S (tmp0, buf1[1], 0x7650); + break; + case 21: tmp1 = __byte_perm_S (buf1[1], 0, 0x6541); + buf1[1] = __byte_perm_S (tmp0, buf1[1], 0x7604); + break; + case 22: tmp1 = __byte_perm_S (buf1[1], 0, 0x6542); + buf1[1] = __byte_perm_S (tmp0, buf1[1], 0x7054); + break; + case 23: tmp1 = __byte_perm_S (buf1[1], 0, 0x6543); + buf1[1] = __byte_perm_S (tmp0, buf1[1], 0x0654); + break; + case 24: tmp1 = __byte_perm_S (buf1[2], 0, 0x6540); + buf1[2] = __byte_perm_S (tmp0, buf1[2], 0x7650); + break; + case 25: tmp1 = __byte_perm_S (buf1[2], 0, 0x6541); + buf1[2] = __byte_perm_S (tmp0, buf1[2], 0x7604); + break; + case 26: tmp1 = __byte_perm_S (buf1[2], 0, 0x6542); + buf1[2] = __byte_perm_S (tmp0, buf1[2], 0x7054); + break; + case 27: tmp1 = __byte_perm_S (buf1[2], 0, 0x6543); + buf1[2] = __byte_perm_S (tmp0, buf1[2], 0x0654); + break; + case 28: tmp1 = __byte_perm_S (buf1[3], 0, 0x6540); + buf1[3] = __byte_perm_S (tmp0, buf1[3], 0x7650); + break; + case 29: tmp1 = __byte_perm_S (buf1[3], 0, 0x6541); + buf1[3] = __byte_perm_S (tmp0, buf1[3], 0x7604); + break; + case 30: tmp1 = __byte_perm_S (buf1[3], 0, 0x6542); + buf1[3] = __byte_perm_S (tmp0, buf1[3], 0x7054); + break; + case 31: tmp1 = __byte_perm_S (buf1[3], 0, 0x6543); + buf1[3] = __byte_perm_S (tmp0, buf1[3], 0x0654); + break; + } + + switch (p0) + { + case 0: buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7650); + break; + case 1: buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7604); + break; + case 2: buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7054); + break; + case 3: buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x0654); + break; + case 4: buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7650); + break; + case 5: buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7604); + break; + case 6: buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7054); + break; + case 7: buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x0654); + break; + case 8: buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7650); + break; + case 9: buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7604); + break; + case 10: buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7054); + break; + case 11: buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x0654); + break; + case 12: buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7650); + break; + case 13: buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7604); + break; + case 14: buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7054); + break; + case 15: buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x0654); + break; + case 16: buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7650); + break; + case 17: buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7604); + break; + case 18: buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7054); + break; + case 19: buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x0654); + break; + case 20: buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7650); + break; + case 21: buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7604); + break; + case 22: buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7054); + break; + case 23: buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x0654); + break; + case 24: buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7650); + break; + case 25: buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7604); + break; + case 26: buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7054); + break; + case 27: buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x0654); + break; + case 28: buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7650); + break; + case 29: buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7604); + break; + case 30: buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7054); + break; + case 31: buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x0654); + break; + } + #endif + + #if defined IS_AMD || defined IS_GENERIC + switch (p0) + { + case 0: tmp0 = (buf0[0] >> 0) & 0xFF; + break; + case 1: tmp0 = (buf0[0] >> 8) & 0xFF; + break; + case 2: tmp0 = (buf0[0] >> 16) & 0xFF; + break; + case 3: tmp0 = (buf0[0] >> 24) & 0xFF; + break; + case 4: tmp0 = (buf0[1] >> 0) & 0xFF; + break; + case 5: tmp0 = (buf0[1] >> 8) & 0xFF; + break; + case 6: tmp0 = (buf0[1] >> 16) & 0xFF; + break; + case 7: tmp0 = (buf0[1] >> 24) & 0xFF; + break; + case 8: tmp0 = (buf0[2] >> 0) & 0xFF; + break; + case 9: tmp0 = (buf0[2] >> 8) & 0xFF; + break; + case 10: tmp0 = (buf0[2] >> 16) & 0xFF; + break; + case 11: tmp0 = (buf0[2] >> 24) & 0xFF; + break; + case 12: tmp0 = (buf0[3] >> 0) & 0xFF; + break; + case 13: tmp0 = (buf0[3] >> 8) & 0xFF; + break; + case 14: tmp0 = (buf0[3] >> 16) & 0xFF; + break; + case 15: tmp0 = (buf0[3] >> 24) & 0xFF; + break; + case 16: tmp0 = (buf1[0] >> 0) & 0xFF; + break; + case 17: tmp0 = (buf1[0] >> 8) & 0xFF; + break; + case 18: tmp0 = (buf1[0] >> 16) & 0xFF; + break; + case 19: tmp0 = (buf1[0] >> 24) & 0xFF; + break; + case 20: tmp0 = (buf1[1] >> 0) & 0xFF; + break; + case 21: tmp0 = (buf1[1] >> 8) & 0xFF; + break; + case 22: tmp0 = (buf1[1] >> 16) & 0xFF; + break; + case 23: tmp0 = (buf1[1] >> 24) & 0xFF; + break; + case 24: tmp0 = (buf1[2] >> 0) & 0xFF; + break; + case 25: tmp0 = (buf1[2] >> 8) & 0xFF; + break; + case 26: tmp0 = (buf1[2] >> 16) & 0xFF; + break; + case 27: tmp0 = (buf1[2] >> 24) & 0xFF; + break; + case 28: tmp0 = (buf1[3] >> 0) & 0xFF; + break; + case 29: tmp0 = (buf1[3] >> 8) & 0xFF; + break; + case 30: tmp0 = (buf1[3] >> 16) & 0xFF; + break; + case 31: tmp0 = (buf1[3] >> 24) & 0xFF; + break; + } + + switch (p1) + { + case 0: tmp1 = (buf0[0] >> 0) & 0xff; + buf0[0] = (buf0[0] & 0xffffff00) | tmp0 << 0; + break; + case 1: tmp1 = (buf0[0] >> 8) & 0xff; + buf0[0] = (buf0[0] & 0xffff00ff) | tmp0 << 8; + break; + case 2: tmp1 = (buf0[0] >> 16) & 0xff; + buf0[0] = (buf0[0] & 0xff00ffff) | tmp0 << 16; + break; + case 3: tmp1 = (buf0[0] >> 24) & 0xff; + buf0[0] = (buf0[0] & 0x00ffffff) | tmp0 << 24; + break; + case 4: tmp1 = (buf0[1] >> 0) & 0xff; + buf0[1] = (buf0[1] & 0xffffff00) | tmp0 << 0; + break; + case 5: tmp1 = (buf0[1] >> 8) & 0xff; + buf0[1] = (buf0[1] & 0xffff00ff) | tmp0 << 8; + break; + case 6: tmp1 = (buf0[1] >> 16) & 0xff; + buf0[1] = (buf0[1] & 0xff00ffff) | tmp0 << 16; + break; + case 7: tmp1 = (buf0[1] >> 24) & 0xff; + buf0[1] = (buf0[1] & 0x00ffffff) | tmp0 << 24; + break; + case 8: tmp1 = (buf0[2] >> 0) & 0xff; + buf0[2] = (buf0[2] & 0xffffff00) | tmp0 << 0; + break; + case 9: tmp1 = (buf0[2] >> 8) & 0xff; + buf0[2] = (buf0[2] & 0xffff00ff) | tmp0 << 8; + break; + case 10: tmp1 = (buf0[2] >> 16) & 0xff; + buf0[2] = (buf0[2] & 0xff00ffff) | tmp0 << 16; + break; + case 11: tmp1 = (buf0[2] >> 24) & 0xff; + buf0[2] = (buf0[2] & 0x00ffffff) | tmp0 << 24; + break; + case 12: tmp1 = (buf0[3] >> 0) & 0xff; + buf0[3] = (buf0[3] & 0xffffff00) | tmp0 << 0; + break; + case 13: tmp1 = (buf0[3] >> 8) & 0xff; + buf0[3] = (buf0[3] & 0xffff00ff) | tmp0 << 8; + break; + case 14: tmp1 = (buf0[3] >> 16) & 0xff; + buf0[3] = (buf0[3] & 0xff00ffff) | tmp0 << 16; + break; + case 15: tmp1 = (buf0[3] >> 24) & 0xff; + buf0[3] = (buf0[3] & 0x00ffffff) | tmp0 << 24; + break; + case 16: tmp1 = (buf1[0] >> 0) & 0xff; + buf1[0] = (buf1[0] & 0xffffff00) | tmp0 << 0; + break; + case 17: tmp1 = (buf1[0] >> 8) & 0xff; + buf1[0] = (buf1[0] & 0xffff00ff) | tmp0 << 8; + break; + case 18: tmp1 = (buf1[0] >> 16) & 0xff; + buf1[0] = (buf1[0] & 0xff00ffff) | tmp0 << 16; + break; + case 19: tmp1 = (buf1[0] >> 24) & 0xff; + buf1[0] = (buf1[0] & 0x00ffffff) | tmp0 << 24; + break; + case 20: tmp1 = (buf1[1] >> 0) & 0xff; + buf1[1] = (buf1[1] & 0xffffff00) | tmp0 << 0; + break; + case 21: tmp1 = (buf1[1] >> 8) & 0xff; + buf1[1] = (buf1[1] & 0xffff00ff) | tmp0 << 8; + break; + case 22: tmp1 = (buf1[1] >> 16) & 0xff; + buf1[1] = (buf1[1] & 0xff00ffff) | tmp0 << 16; + break; + case 23: tmp1 = (buf1[1] >> 24) & 0xff; + buf1[1] = (buf1[1] & 0x00ffffff) | tmp0 << 24; + break; + case 24: tmp1 = (buf1[2] >> 0) & 0xff; + buf1[2] = (buf1[2] & 0xffffff00) | tmp0 << 0; + break; + case 25: tmp1 = (buf1[2] >> 8) & 0xff; + buf1[2] = (buf1[2] & 0xffff00ff) | tmp0 << 8; + break; + case 26: tmp1 = (buf1[2] >> 16) & 0xff; + buf1[2] = (buf1[2] & 0xff00ffff) | tmp0 << 16; + break; + case 27: tmp1 = (buf1[2] >> 24) & 0xff; + buf1[2] = (buf1[2] & 0x00ffffff) | tmp0 << 24; + break; + case 28: tmp1 = (buf1[3] >> 0) & 0xff; + buf1[3] = (buf1[3] & 0xffffff00) | tmp0 << 0; + break; + case 29: tmp1 = (buf1[3] >> 8) & 0xff; + buf1[3] = (buf1[3] & 0xffff00ff) | tmp0 << 8; + break; + case 30: tmp1 = (buf1[3] >> 16) & 0xff; + buf1[3] = (buf1[3] & 0xff00ffff) | tmp0 << 16; + break; + case 31: tmp1 = (buf1[3] >> 24) & 0xff; + buf1[3] = (buf1[3] & 0x00ffffff) | tmp0 << 24; + break; + } + + switch (p0) + { + case 0: buf0[0] = (buf0[0] & 0xffffff00) | tmp1 << 0; + break; + case 1: buf0[0] = (buf0[0] & 0xffff00ff) | tmp1 << 8; + break; + case 2: buf0[0] = (buf0[0] & 0xff00ffff) | tmp1 << 16; + break; + case 3: buf0[0] = (buf0[0] & 0x00ffffff) | tmp1 << 24; + break; + case 4: buf0[1] = (buf0[1] & 0xffffff00) | tmp1 << 0; + break; + case 5: buf0[1] = (buf0[1] & 0xffff00ff) | tmp1 << 8; + break; + case 6: buf0[1] = (buf0[1] & 0xff00ffff) | tmp1 << 16; + break; + case 7: buf0[1] = (buf0[1] & 0x00ffffff) | tmp1 << 24; + break; + case 8: buf0[2] = (buf0[2] & 0xffffff00) | tmp1 << 0; + break; + case 9: buf0[2] = (buf0[2] & 0xffff00ff) | tmp1 << 8; + break; + case 10: buf0[2] = (buf0[2] & 0xff00ffff) | tmp1 << 16; + break; + case 11: buf0[2] = (buf0[2] & 0x00ffffff) | tmp1 << 24; + break; + case 12: buf0[3] = (buf0[3] & 0xffffff00) | tmp1 << 0; + break; + case 13: buf0[3] = (buf0[3] & 0xffff00ff) | tmp1 << 8; + break; + case 14: buf0[3] = (buf0[3] & 0xff00ffff) | tmp1 << 16; + break; + case 15: buf0[3] = (buf0[3] & 0x00ffffff) | tmp1 << 24; + break; + case 16: buf1[0] = (buf1[0] & 0xffffff00) | tmp1 << 0; + break; + case 17: buf1[0] = (buf1[0] & 0xffff00ff) | tmp1 << 8; + break; + case 18: buf1[0] = (buf1[0] & 0xff00ffff) | tmp1 << 16; + break; + case 19: buf1[0] = (buf1[0] & 0x00ffffff) | tmp1 << 24; + break; + case 20: buf1[1] = (buf1[1] & 0xffffff00) | tmp1 << 0; + break; + case 21: buf1[1] = (buf1[1] & 0xffff00ff) | tmp1 << 8; + break; + case 22: buf1[1] = (buf1[1] & 0xff00ffff) | tmp1 << 16; + break; + case 23: buf1[1] = (buf1[1] & 0x00ffffff) | tmp1 << 24; + break; + case 24: buf1[2] = (buf1[2] & 0xffffff00) | tmp1 << 0; + break; + case 25: buf1[2] = (buf1[2] & 0xffff00ff) | tmp1 << 8; + break; + case 26: buf1[2] = (buf1[2] & 0xff00ffff) | tmp1 << 16; + break; + case 27: buf1[2] = (buf1[2] & 0x00ffffff) | tmp1 << 24; + break; + case 28: buf1[3] = (buf1[3] & 0xffffff00) | tmp1 << 0; + break; + case 29: buf1[3] = (buf1[3] & 0xffff00ff) | tmp1 << 8; + break; + case 30: buf1[3] = (buf1[3] & 0xff00ffff) | tmp1 << 16; + break; + case 31: buf1[3] = (buf1[3] & 0x00ffffff) | tmp1 << 24; + break; + } + #endif + + return in_len; +} + +inline u32 rule_op_mangle_chr_shiftl (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + const u32 mr = 0xffu << ((p0 & 3) * 8); + const u32 ml = ~mr; + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) << 1) & mr); break; + case 1: buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) << 1) & mr); break; + case 2: buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) << 1) & mr); break; + case 3: buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) << 1) & mr); break; + case 4: buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) << 1) & mr); break; + case 5: buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) << 1) & mr); break; + case 6: buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) << 1) & mr); break; + case 7: buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) << 1) & mr); break; + } + + return in_len; +} + +inline u32 rule_op_mangle_chr_shiftr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + const u32 mr = 0xffu << ((p0 & 3) * 8); + const u32 ml = ~mr; + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) >> 1) & mr); break; + case 1: buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) >> 1) & mr); break; + case 2: buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) >> 1) & mr); break; + case 3: buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) >> 1) & mr); break; + case 4: buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) >> 1) & mr); break; + case 5: buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) >> 1) & mr); break; + case 6: buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) >> 1) & mr); break; + case 7: buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) >> 1) & mr); break; + } + + return in_len; +} + +inline u32 rule_op_mangle_chr_incr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + const u32 mr = 0xffu << ((p0 & 3) * 8); + const u32 ml = ~mr; + + const u32 n = 0x01010101 & mr; + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) + n) & mr); break; + case 1: buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) + n) & mr); break; + case 2: buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) + n) & mr); break; + case 3: buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) + n) & mr); break; + case 4: buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) + n) & mr); break; + case 5: buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) + n) & mr); break; + case 6: buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) + n) & mr); break; + case 7: buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) + n) & mr); break; + } + + return in_len; +} + +inline u32 rule_op_mangle_chr_decr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 >= in_len) return (in_len); + + const u32 mr = 0xffu << ((p0 & 3) * 8); + const u32 ml = ~mr; + + const u32 n = 0x01010101 & mr; + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) - n) & mr); break; + case 1: buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) - n) & mr); break; + case 2: buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) - n) & mr); break; + case 3: buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) - n) & mr); break; + case 4: buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) - n) & mr); break; + case 5: buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) - n) & mr); break; + case 6: buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) - n) & mr); break; + case 7: buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) - n) & mr); break; + } + + return in_len; +} + +inline u32 rule_op_mangle_replace_np1 (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if ((p0 + 1) >= in_len) return (in_len); + + u32 tib40[4]; + u32 tib41[4]; + + lshift_block (buf0, buf1, tib40, tib41); + + const u32 mr = 0xffu << ((p0 & 3) * 8); + const u32 ml = ~mr; + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & ml) | (tib40[0] & mr); break; + case 1: buf0[1] = (buf0[1] & ml) | (tib40[1] & mr); break; + case 2: buf0[2] = (buf0[2] & ml) | (tib40[2] & mr); break; + case 3: buf0[3] = (buf0[3] & ml) | (tib40[3] & mr); break; + case 4: buf1[0] = (buf1[0] & ml) | (tib41[0] & mr); break; + case 5: buf1[1] = (buf1[1] & ml) | (tib41[1] & mr); break; + case 6: buf1[2] = (buf1[2] & ml) | (tib41[2] & mr); break; + case 7: buf1[3] = (buf1[3] & ml) | (tib41[3] & mr); break; + } + + return in_len; +} + +inline u32 rule_op_mangle_replace_nm1 (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 == 0) return (in_len); + + if (p0 >= in_len) return (in_len); + + u32 tib40[4]; + u32 tib41[4]; + + rshift_block (buf0, buf1, tib40, tib41); + + const u32 mr = 0xffu << ((p0 & 3) * 8); + const u32 ml = ~mr; + + switch (p0 / 4) + { + case 0: buf0[0] = (buf0[0] & ml) | (tib40[0] & mr); break; + case 1: buf0[1] = (buf0[1] & ml) | (tib40[1] & mr); break; + case 2: buf0[2] = (buf0[2] & ml) | (tib40[2] & mr); break; + case 3: buf0[3] = (buf0[3] & ml) | (tib40[3] & mr); break; + case 4: buf1[0] = (buf1[0] & ml) | (tib41[0] & mr); break; + case 5: buf1[1] = (buf1[1] & ml) | (tib41[1] & mr); break; + case 6: buf1[2] = (buf1[2] & ml) | (tib41[2] & mr); break; + case 7: buf1[3] = (buf1[3] & ml) | (tib41[3] & mr); break; + } + + return in_len; +} + +inline u32 rule_op_mangle_dupeblock_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 > in_len) return (in_len); + + if ((in_len + p0) >= 32) return (in_len); + + u32 out_len = in_len; + + u32 tib40[4]; + u32 tib41[4]; + + tib40[0] = buf0[0]; + tib40[1] = buf0[1]; + tib40[2] = buf0[2]; + tib40[3] = buf0[3]; + tib41[0] = buf1[0]; + tib41[1] = buf1[1]; + tib41[2] = buf1[2]; + tib41[3] = buf1[3]; + + truncate_right (tib40, tib41, p0); + + rshift_block_N (buf0, buf1, buf0, buf1, p0); + + buf0[0] |= tib40[0]; + buf0[1] |= tib40[1]; + buf0[2] |= tib40[2]; + buf0[3] |= tib40[3]; + buf1[0] |= tib41[0]; + buf1[1] |= tib41[1]; + buf1[2] |= tib41[2]; + buf1[3] |= tib41[3]; + + out_len += p0; + + return out_len; +} + +inline u32 rule_op_mangle_dupeblock_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + if (p0 > in_len) return (in_len); + + if ((in_len + p0) >= 32) return (in_len); + + u32 out_len = in_len; + + u32 tib40[4]; + u32 tib41[4]; + + rshift_block_N (buf0, buf1, tib40, tib41, p0); + + truncate_left (tib40, tib41, out_len); + + buf0[0] |= tib40[0]; + buf0[1] |= tib40[1]; + buf0[2] |= tib40[2]; + buf0[3] |= tib40[3]; + buf1[0] |= tib41[0]; + buf1[1] |= tib41[1]; + buf1[2] |= tib41[2]; + buf1[3] |= tib41[3]; + + out_len += p0; + + return out_len; +} + +inline u32 rule_op_mangle_title (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + buf0[0] |= (generate_cmask (buf0[0])); + buf0[1] |= (generate_cmask (buf0[1])); + buf0[2] |= (generate_cmask (buf0[2])); + buf0[3] |= (generate_cmask (buf0[3])); + buf1[0] |= (generate_cmask (buf1[0])); + buf1[1] |= (generate_cmask (buf1[1])); + buf1[2] |= (generate_cmask (buf1[2])); + buf1[3] |= (generate_cmask (buf1[3])); + + #ifdef IS_NV + buf0[0] &= ~(0x00000020 & generate_cmask (buf0[0])); + + for (u32 i = 0; i < in_len; i++) + { + u32 tmp0; + u32 tmp1; + + switch (i) + { + case 0: tmp0 = __byte_perm_S (buf0[0], 0, 0x6540); + tmp1 = ~(0x00002000 & generate_cmask (buf0[0])); break; + case 1: tmp0 = __byte_perm_S (buf0[0], 0, 0x6541); + tmp1 = ~(0x00200000 & generate_cmask (buf0[0])); break; + case 2: tmp0 = __byte_perm_S (buf0[0], 0, 0x6542); + tmp1 = ~(0x20000000 & generate_cmask (buf0[0])); break; + case 3: tmp0 = __byte_perm_S (buf0[0], 0, 0x6543); + tmp1 = ~(0x00000020 & generate_cmask (buf0[1])); break; + case 4: tmp0 = __byte_perm_S (buf0[1], 0, 0x6540); + tmp1 = ~(0x00002000 & generate_cmask (buf0[1])); break; + case 5: tmp0 = __byte_perm_S (buf0[1], 0, 0x6541); + tmp1 = ~(0x00200000 & generate_cmask (buf0[1])); break; + case 6: tmp0 = __byte_perm_S (buf0[1], 0, 0x6542); + tmp1 = ~(0x20000000 & generate_cmask (buf0[1])); break; + case 7: tmp0 = __byte_perm_S (buf0[1], 0, 0x6543); + tmp1 = ~(0x00000020 & generate_cmask (buf0[2])); break; + case 8: tmp0 = __byte_perm_S (buf0[2], 0, 0x6540); + tmp1 = ~(0x00002000 & generate_cmask (buf0[2])); break; + case 9: tmp0 = __byte_perm_S (buf0[2], 0, 0x6541); + tmp1 = ~(0x00200000 & generate_cmask (buf0[2])); break; + case 10: tmp0 = __byte_perm_S (buf0[2], 0, 0x6542); + tmp1 = ~(0x20000000 & generate_cmask (buf0[2])); break; + case 11: tmp0 = __byte_perm_S (buf0[2], 0, 0x6543); + tmp1 = ~(0x00000020 & generate_cmask (buf0[3])); break; + case 12: tmp0 = __byte_perm_S (buf0[3], 0, 0x6540); + tmp1 = ~(0x00002000 & generate_cmask (buf0[3])); break; + case 13: tmp0 = __byte_perm_S (buf0[3], 0, 0x6541); + tmp1 = ~(0x00200000 & generate_cmask (buf0[3])); break; + case 14: tmp0 = __byte_perm_S (buf0[3], 0, 0x6542); + tmp1 = ~(0x20000000 & generate_cmask (buf0[3])); break; + case 15: tmp0 = __byte_perm_S (buf0[3], 0, 0x6543); + tmp1 = ~(0x00000020 & generate_cmask (buf1[0])); break; + case 16: tmp0 = __byte_perm_S (buf1[0], 0, 0x6540); + tmp1 = ~(0x00002000 & generate_cmask (buf1[0])); break; + case 17: tmp0 = __byte_perm_S (buf1[0], 0, 0x6541); + tmp1 = ~(0x00200000 & generate_cmask (buf1[0])); break; + case 18: tmp0 = __byte_perm_S (buf1[0], 0, 0x6542); + tmp1 = ~(0x20000000 & generate_cmask (buf1[0])); break; + case 19: tmp0 = __byte_perm_S (buf1[0], 0, 0x6543); + tmp1 = ~(0x00000020 & generate_cmask (buf1[1])); break; + case 20: tmp0 = __byte_perm_S (buf1[1], 0, 0x6540); + tmp1 = ~(0x00002000 & generate_cmask (buf1[1])); break; + case 21: tmp0 = __byte_perm_S (buf1[1], 0, 0x6541); + tmp1 = ~(0x00200000 & generate_cmask (buf1[1])); break; + case 22: tmp0 = __byte_perm_S (buf1[1], 0, 0x6542); + tmp1 = ~(0x20000000 & generate_cmask (buf1[1])); break; + case 23: tmp0 = __byte_perm_S (buf1[1], 0, 0x6543); + tmp1 = ~(0x00000020 & generate_cmask (buf1[2])); break; + case 24: tmp0 = __byte_perm_S (buf1[2], 0, 0x6540); + tmp1 = ~(0x00002000 & generate_cmask (buf1[2])); break; + case 25: tmp0 = __byte_perm_S (buf1[2], 0, 0x6541); + tmp1 = ~(0x00200000 & generate_cmask (buf1[2])); break; + case 26: tmp0 = __byte_perm_S (buf1[2], 0, 0x6542); + tmp1 = ~(0x20000000 & generate_cmask (buf1[2])); break; + case 27: tmp0 = __byte_perm_S (buf1[2], 0, 0x6543); + tmp1 = ~(0x00000020 & generate_cmask (buf1[3])); break; + case 28: tmp0 = __byte_perm_S (buf1[3], 0, 0x6540); + tmp1 = ~(0x00002000 & generate_cmask (buf1[3])); break; + case 29: tmp0 = __byte_perm_S (buf1[3], 0, 0x6541); + tmp1 = ~(0x00200000 & generate_cmask (buf1[3])); break; + case 30: tmp0 = __byte_perm_S (buf1[3], 0, 0x6542); + tmp1 = ~(0x20000000 & generate_cmask (buf1[3])); break; + } + + if (i < 3) + { + if (tmp0 == ' ') buf0[0] &= tmp1 ; + } + else if (i < 7) + { + if (tmp0 == ' ') buf0[1] &= tmp1 ; + } + else if (i < 11) + { + if (tmp0 == ' ') buf0[2] &= tmp1 ; + } + else if (i < 15) + { + if (tmp0 == ' ') buf0[3] &= tmp1 ; + } + else if (i < 19) + { + if (tmp0 == ' ') buf1[0] &= tmp1 ; + } + else if (i < 23) + { + if (tmp0 == ' ') buf1[1] &= tmp1 ; + } + else if (i < 27) + { + if (tmp0 == ' ') buf1[2] &= tmp1 ; + } + else if (i < 31) + { + if (tmp0 == ' ') buf1[3] &= tmp1 ; + } + } + #endif + + #if defined IS_AMD || defined IS_GENERIC + u32 tib40[4]; + u32 tib41[4]; + + const uchar4 tmp0 = (uchar4) (' '); + const uchar4 tmp1 = (uchar4) (0x00); + const uchar4 tmp2 = (uchar4) (0xff); + + uchar4 tmp; + + tmp = as_uchar4 (buf0[0]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[0] = as_uint (tmp); + tmp = as_uchar4 (buf0[1]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[1] = as_uint (tmp); + tmp = as_uchar4 (buf0[2]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[2] = as_uint (tmp); + tmp = as_uchar4 (buf0[3]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[3] = as_uint (tmp); + tmp = as_uchar4 (buf1[0]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[0] = as_uint (tmp); + tmp = as_uchar4 (buf1[1]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[1] = as_uint (tmp); + tmp = as_uchar4 (buf1[2]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[2] = as_uint (tmp); + tmp = as_uchar4 (buf1[3]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[3] = as_uint (tmp); + + rshift_block (tib40, tib41, tib40, tib41); tib40[0] |= 0xff; + + buf0[0] &= ~(generate_cmask (buf0[0]) & tib40[0]); + buf0[1] &= ~(generate_cmask (buf0[1]) & tib40[1]); + buf0[2] &= ~(generate_cmask (buf0[2]) & tib40[2]); + buf0[3] &= ~(generate_cmask (buf0[3]) & tib40[3]); + buf1[0] &= ~(generate_cmask (buf1[0]) & tib41[0]); + buf1[1] &= ~(generate_cmask (buf1[1]) & tib41[1]); + buf1[2] &= ~(generate_cmask (buf1[2]) & tib41[2]); + buf1[3] &= ~(generate_cmask (buf1[3]) & tib41[3]); + #endif + + return in_len; +} + +inline u32 apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) +{ + u32 out_len = in_len; + + switch (name) + { + case RULE_OP_MANGLE_LREST: out_len = rule_op_mangle_lrest (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_UREST: out_len = rule_op_mangle_urest (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_LREST_UFIRST: out_len = rule_op_mangle_lrest_ufirst (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_UREST_LFIRST: out_len = rule_op_mangle_urest_lfirst (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_TREST: out_len = rule_op_mangle_trest (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_TOGGLE_AT: out_len = rule_op_mangle_toggle_at (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_REVERSE: out_len = rule_op_mangle_reverse (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DUPEWORD: out_len = rule_op_mangle_dupeword (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DUPEWORD_TIMES: out_len = rule_op_mangle_dupeword_times (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_REFLECT: out_len = rule_op_mangle_reflect (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_APPEND: out_len = rule_op_mangle_append (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_PREPEND: out_len = rule_op_mangle_prepend (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_ROTATE_LEFT: out_len = rule_op_mangle_rotate_left (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_ROTATE_RIGHT: out_len = rule_op_mangle_rotate_right (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DELETE_FIRST: out_len = rule_op_mangle_delete_first (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DELETE_LAST: out_len = rule_op_mangle_delete_last (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DELETE_AT: out_len = rule_op_mangle_delete_at (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_EXTRACT: out_len = rule_op_mangle_extract (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_OMIT: out_len = rule_op_mangle_omit (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_INSERT: out_len = rule_op_mangle_insert (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_OVERSTRIKE: out_len = rule_op_mangle_overstrike (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_TRUNCATE_AT: out_len = rule_op_mangle_truncate_at (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_REPLACE: out_len = rule_op_mangle_replace (p0, p1, buf0, buf1, out_len); break; + //case RULE_OP_MANGLE_PURGECHAR: out_len = rule_op_mangle_purgechar (p0, p1, buf0, buf1, out_len); break; + //case RULE_OP_MANGLE_TOGGLECASE_REC: out_len = rule_op_mangle_togglecase_rec (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DUPECHAR_FIRST: out_len = rule_op_mangle_dupechar_first (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DUPECHAR_LAST: out_len = rule_op_mangle_dupechar_last (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DUPECHAR_ALL: out_len = rule_op_mangle_dupechar_all (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_SWITCH_FIRST: out_len = rule_op_mangle_switch_first (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_SWITCH_LAST: out_len = rule_op_mangle_switch_last (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_SWITCH_AT: out_len = rule_op_mangle_switch_at (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_CHR_SHIFTL: out_len = rule_op_mangle_chr_shiftl (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_CHR_SHIFTR: out_len = rule_op_mangle_chr_shiftr (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_CHR_INCR: out_len = rule_op_mangle_chr_incr (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_CHR_DECR: out_len = rule_op_mangle_chr_decr (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_REPLACE_NP1: out_len = rule_op_mangle_replace_np1 (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_REPLACE_NM1: out_len = rule_op_mangle_replace_nm1 (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DUPEBLOCK_FIRST: out_len = rule_op_mangle_dupeblock_first (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_DUPEBLOCK_LAST: out_len = rule_op_mangle_dupeblock_last (p0, p1, buf0, buf1, out_len); break; + case RULE_OP_MANGLE_TITLE: out_len = rule_op_mangle_title (p0, p1, buf0, buf1, out_len); break; + } + + return out_len; +} + +inline u32 apply_rules (const __global u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len) +{ + u32 out_len = len; + + for (u32 i = 0; cmds[i] != 0; i++) + { + const u32 cmd = cmds[i]; + + const u32 name = (cmd >> 0) & 0xff; + const u32 p0 = (cmd >> 8) & 0xff; + const u32 p1 = (cmd >> 16) & 0xff; + + out_len = apply_rule (name, p0, p1, buf0, buf1, out_len); + } + + return out_len; +} + +inline u32x apply_rules_vect (const u32 pw_buf0[4], const u32 pw_buf1[4], const u32 pw_len, const __global kernel_rule_t *rules_buf, const u32 il_pos, u32x w0[4], u32x w1[4]) +{ + #if VECT_SIZE == 1 + + w0[0] = pw_buf0[0]; + w0[1] = pw_buf0[1]; + w0[2] = pw_buf0[2]; + w0[3] = pw_buf0[3]; + w1[0] = pw_buf1[0]; + w1[1] = pw_buf1[1]; + w1[2] = pw_buf1[2]; + w1[3] = pw_buf1[3]; + + return apply_rules (rules_buf[il_pos].cmds, w0, w1, pw_len); + + #else + + u32x out_len = 0; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < VECT_SIZE; i++) + { + u32 tmp0[4]; + u32 tmp1[4]; + + tmp0[0] = pw_buf0[0]; + tmp0[1] = pw_buf0[1]; + tmp0[2] = pw_buf0[2]; + tmp0[3] = pw_buf0[3]; + tmp1[0] = pw_buf1[0]; + tmp1[1] = pw_buf1[1]; + tmp1[2] = pw_buf1[2]; + tmp1[3] = pw_buf1[3]; + + const u32 tmp_len = apply_rules (rules_buf[il_pos + i].cmds, tmp0, tmp1, pw_len); + + switch (i) + { + #if VECT_SIZE >= 2 + case 0: + w0[0].s0 = tmp0[0]; + w0[1].s0 = tmp0[1]; + w0[2].s0 = tmp0[2]; + w0[3].s0 = tmp0[3]; + w1[0].s0 = tmp1[0]; + w1[1].s0 = tmp1[1]; + w1[2].s0 = tmp1[2]; + w1[3].s0 = tmp1[3]; + out_len.s0 = tmp_len; + break; + + case 1: + w0[0].s1 = tmp0[0]; + w0[1].s1 = tmp0[1]; + w0[2].s1 = tmp0[2]; + w0[3].s1 = tmp0[3]; + w1[0].s1 = tmp1[0]; + w1[1].s1 = tmp1[1]; + w1[2].s1 = tmp1[2]; + w1[3].s1 = tmp1[3]; + out_len.s1 = tmp_len; + break; + #endif + + #if VECT_SIZE >= 4 + case 2: + w0[0].s2 = tmp0[0]; + w0[1].s2 = tmp0[1]; + w0[2].s2 = tmp0[2]; + w0[3].s2 = tmp0[3]; + w1[0].s2 = tmp1[0]; + w1[1].s2 = tmp1[1]; + w1[2].s2 = tmp1[2]; + w1[3].s2 = tmp1[3]; + out_len.s2 = tmp_len; + break; + + case 3: + w0[0].s3 = tmp0[0]; + w0[1].s3 = tmp0[1]; + w0[2].s3 = tmp0[2]; + w0[3].s3 = tmp0[3]; + w1[0].s3 = tmp1[0]; + w1[1].s3 = tmp1[1]; + w1[2].s3 = tmp1[2]; + w1[3].s3 = tmp1[3]; + out_len.s3 = tmp_len; + break; + #endif + + #if VECT_SIZE >= 8 + case 4: + w0[0].s4 = tmp0[0]; + w0[1].s4 = tmp0[1]; + w0[2].s4 = tmp0[2]; + w0[3].s4 = tmp0[3]; + w1[0].s4 = tmp1[0]; + w1[1].s4 = tmp1[1]; + w1[2].s4 = tmp1[2]; + w1[3].s4 = tmp1[3]; + out_len.s4 = tmp_len; + break; + + case 5: + w0[0].s5 = tmp0[0]; + w0[1].s5 = tmp0[1]; + w0[2].s5 = tmp0[2]; + w0[3].s5 = tmp0[3]; + w1[0].s5 = tmp1[0]; + w1[1].s5 = tmp1[1]; + w1[2].s5 = tmp1[2]; + w1[3].s5 = tmp1[3]; + out_len.s5 = tmp_len; + break; + + case 6: + w0[0].s6 = tmp0[0]; + w0[1].s6 = tmp0[1]; + w0[2].s6 = tmp0[2]; + w0[3].s6 = tmp0[3]; + w1[0].s6 = tmp1[0]; + w1[1].s6 = tmp1[1]; + w1[2].s6 = tmp1[2]; + w1[3].s6 = tmp1[3]; + out_len.s6 = tmp_len; + break; + + case 7: + w0[0].s7 = tmp0[0]; + w0[1].s7 = tmp0[1]; + w0[2].s7 = tmp0[2]; + w0[3].s7 = tmp0[3]; + w1[0].s7 = tmp1[0]; + w1[1].s7 = tmp1[1]; + w1[2].s7 = tmp1[2]; + w1[3].s7 = tmp1[3]; + out_len.s7 = tmp_len; + break; + #endif + + #if VECT_SIZE >= 16 + case 8: + w0[0].s8 = tmp0[0]; + w0[1].s8 = tmp0[1]; + w0[2].s8 = tmp0[2]; + w0[3].s8 = tmp0[3]; + w1[0].s8 = tmp1[0]; + w1[1].s8 = tmp1[1]; + w1[2].s8 = tmp1[2]; + w1[3].s8 = tmp1[3]; + out_len.s8 = tmp_len; + break; + + case 9: + w0[0].s9 = tmp0[0]; + w0[1].s9 = tmp0[1]; + w0[2].s9 = tmp0[2]; + w0[3].s9 = tmp0[3]; + w1[0].s9 = tmp1[0]; + w1[1].s9 = tmp1[1]; + w1[2].s9 = tmp1[2]; + w1[3].s9 = tmp1[3]; + out_len.s9 = tmp_len; + break; + + case 10: + w0[0].sa = tmp0[0]; + w0[1].sa = tmp0[1]; + w0[2].sa = tmp0[2]; + w0[3].sa = tmp0[3]; + w1[0].sa = tmp1[0]; + w1[1].sa = tmp1[1]; + w1[2].sa = tmp1[2]; + w1[3].sa = tmp1[3]; + out_len.sa = tmp_len; + break; + + case 11: + w0[0].sb = tmp0[0]; + w0[1].sb = tmp0[1]; + w0[2].sb = tmp0[2]; + w0[3].sb = tmp0[3]; + w1[0].sb = tmp1[0]; + w1[1].sb = tmp1[1]; + w1[2].sb = tmp1[2]; + w1[3].sb = tmp1[3]; + out_len.sb = tmp_len; + break; + + case 12: + w0[0].sc = tmp0[0]; + w0[1].sc = tmp0[1]; + w0[2].sc = tmp0[2]; + w0[3].sc = tmp0[3]; + w1[0].sc = tmp1[0]; + w1[1].sc = tmp1[1]; + w1[2].sc = tmp1[2]; + w1[3].sc = tmp1[3]; + out_len.sc = tmp_len; + break; + + case 13: + w0[0].sd = tmp0[0]; + w0[1].sd = tmp0[1]; + w0[2].sd = tmp0[2]; + w0[3].sd = tmp0[3]; + w1[0].sd = tmp1[0]; + w1[1].sd = tmp1[1]; + w1[2].sd = tmp1[2]; + w1[3].sd = tmp1[3]; + out_len.sd = tmp_len; + break; + + case 14: + w0[0].se = tmp0[0]; + w0[1].se = tmp0[1]; + w0[2].se = tmp0[2]; + w0[3].se = tmp0[3]; + w1[0].se = tmp1[0]; + w1[1].se = tmp1[1]; + w1[2].se = tmp1[2]; + w1[3].se = tmp1[3]; + out_len.se = tmp_len; + break; + + case 15: + w0[0].sf = tmp0[0]; + w0[1].sf = tmp0[1]; + w0[2].sf = tmp0[2]; + w0[3].sf = tmp0[3]; + w1[0].sf = tmp1[0]; + w1[1].sf = tmp1[1]; + w1[2].sf = tmp1[2]; + w1[3].sf = tmp1[3]; + out_len.sf = tmp_len; + break; + #endif + } + } + + return out_len; + + #endif +} diff --git a/OpenCL/inc_rp.h b/OpenCL/inc_rp.h new file mode 100644 index 0000000..17849f2 --- /dev/null +++ b/OpenCL/inc_rp.h @@ -0,0 +1,57 @@ +/** + * Author......: Jens Steube + * License.....: MIT + */ + +#define RULE_OP_MANGLE_NOOP ':' +#define RULE_OP_MANGLE_LREST 'l' +#define RULE_OP_MANGLE_UREST 'u' +#define RULE_OP_MANGLE_LREST_UFIRST 'c' +#define RULE_OP_MANGLE_UREST_LFIRST 'C' +#define RULE_OP_MANGLE_TREST 't' +#define RULE_OP_MANGLE_TOGGLE_AT 'T' +#define RULE_OP_MANGLE_REVERSE 'r' +#define RULE_OP_MANGLE_DUPEWORD 'd' +#define RULE_OP_MANGLE_DUPEWORD_TIMES 'p' +#define RULE_OP_MANGLE_REFLECT 'f' +#define RULE_OP_MANGLE_ROTATE_LEFT '{' +#define RULE_OP_MANGLE_ROTATE_RIGHT '}' +#define RULE_OP_MANGLE_APPEND '$' +#define RULE_OP_MANGLE_PREPEND '^' +#define RULE_OP_MANGLE_DELETE_FIRST '[' +#define RULE_OP_MANGLE_DELETE_LAST ']' +#define RULE_OP_MANGLE_DELETE_AT 'D' +#define RULE_OP_MANGLE_EXTRACT 'x' +#define RULE_OP_MANGLE_OMIT 'O' +#define RULE_OP_MANGLE_INSERT 'i' +#define RULE_OP_MANGLE_OVERSTRIKE 'o' +#define RULE_OP_MANGLE_TRUNCATE_AT '\'' +#define RULE_OP_MANGLE_REPLACE 's' +#define RULE_OP_MANGLE_PURGECHAR '@' +#define RULE_OP_MANGLE_TOGGLECASE_REC 'a' +#define RULE_OP_MANGLE_DUPECHAR_FIRST 'z' +#define RULE_OP_MANGLE_DUPECHAR_LAST 'Z' +#define RULE_OP_MANGLE_DUPECHAR_ALL 'q' + +#define RULE_OP_REJECT_LESS '<' +#define RULE_OP_REJECT_GREATER '>' +#define RULE_OP_REJECT_CONTAIN '!' +#define RULE_OP_REJECT_NOT_CONTAIN '/' +#define RULE_OP_REJECT_EQUAL_FIRST '(' +#define RULE_OP_REJECT_EQUAL_LAST ')' +#define RULE_OP_REJECT_EQUAL_AT '=' +#define RULE_OP_REJECT_CONTAINS '%' + +/* hashcat only */ +#define RULE_OP_MANGLE_SWITCH_FIRST 'k' +#define RULE_OP_MANGLE_SWITCH_LAST 'K' +#define RULE_OP_MANGLE_SWITCH_AT '*' +#define RULE_OP_MANGLE_CHR_SHIFTL 'L' +#define RULE_OP_MANGLE_CHR_SHIFTR 'R' +#define RULE_OP_MANGLE_CHR_INCR '+' +#define RULE_OP_MANGLE_CHR_DECR '-' +#define RULE_OP_MANGLE_REPLACE_NP1 '.' +#define RULE_OP_MANGLE_REPLACE_NM1 ',' +#define RULE_OP_MANGLE_DUPEBLOCK_FIRST 'y' +#define RULE_OP_MANGLE_DUPEBLOCK_LAST 'Y' +#define RULE_OP_MANGLE_TITLE 'E' diff --git a/OpenCL/inc_simd.cl b/OpenCL/inc_simd.cl new file mode 100644 index 0000000..882e320 --- /dev/null +++ b/OpenCL/inc_simd.cl @@ -0,0 +1,1164 @@ +/** + * Authors.....: Jens Steube + * magnum + * + * License.....: MIT + */ + +// vliw1 + +#if VECT_SIZE == 1 + +#define MATCHES_ONE_VV(a,b) ((a) == (b)) +#define MATCHES_ONE_VS(a,b) ((a) == (b)) + +#define COMPARE_S_SIMD(h0,h1,h2,h3) \ +{ \ + if (((h0) == search[0]) && ((h1) == search[1]) && ((h2) == search[2]) && ((h3) == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos); \ + } \ + } \ +} + +#define COMPARE_M_SIMD(h0,h1,h2,h3) \ +{ \ + const u32 digest_tp0[4] = { h0, h1, h2, h3 }; \ + \ + if (check (digest_tp0, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos); \ + } \ + } \ + } \ +} + +#endif + +// vliw2 + +#define vector_accessible(p,c,e) (((p) + (e)) < (c)) + +#if VECT_SIZE == 2 + +#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1)) +#define MATCHES_ONE_VS(a,b) (((a).s0 == (b) ) || ((a).s1 == (b) )) + +#define COMPARE_S_SIMD(h0,h1,h2,h3) \ +{ \ + if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0); \ + } \ + } \ + \ + if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1); \ + } \ + } \ +} + +#define COMPARE_M_SIMD(h0,h1,h2,h3) \ +{ \ + const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 }; \ + const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 }; \ + \ + if (check (digest_tp0, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0); \ + } \ + } \ + } \ + \ + if (check (digest_tp1, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1); \ + } \ + } \ + } \ +} + +#endif + +// vliw4 + +#if VECT_SIZE == 4 + +#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3)) +#define MATCHES_ONE_VS(a,b) (((a).s0 == (b) ) || ((a).s1 == (b) ) || ((a).s2 == (b) ) || ((a).s3 == (b) )) + +#define COMPARE_S_SIMD(h0,h1,h2,h3) \ +{ \ + if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0); \ + } \ + } \ + \ + if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1); \ + } \ + } \ + \ + if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2); \ + } \ + } \ + \ + if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3); \ + } \ + } \ +} + +#define COMPARE_M_SIMD(h0,h1,h2,h3) \ +{ \ + const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 }; \ + const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 }; \ + const u32 digest_tp2[4] = { h0.s2, h1.s2, h2.s2, h3.s2 }; \ + const u32 digest_tp3[4] = { h0.s3, h1.s3, h2.s3, h3.s3 }; \ + \ + if (check (digest_tp0, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0); \ + } \ + } \ + } \ + \ + if (check (digest_tp1, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1); \ + } \ + } \ + } \ + \ + if (check (digest_tp2, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp2, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2); \ + } \ + } \ + } \ + \ + if (check (digest_tp3, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp3, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3); \ + } \ + } \ + } \ +} + +#endif + +// vliw8 + +#if VECT_SIZE == 8 + +#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3) || ((a).s4 == (b).s4) || ((a).s5 == (b).s5) || ((a).s6 == (b).s6) || ((a).s7 == (b).s7)) +#define MATCHES_ONE_VS(a,b) (((a).s0 == (b) ) || ((a).s1 == (b) ) || ((a).s2 == (b) ) || ((a).s3 == (b) ) || ((a).s4 == (b) ) || ((a).s5 == (b) ) || ((a).s6 == (b) ) || ((a).s7 == (b) )) + +#define COMPARE_S_SIMD(h0,h1,h2,h3) \ +{ \ + if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0); \ + } \ + } \ + \ + if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1); \ + } \ + } \ + \ + if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2); \ + } \ + } \ + \ + if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3); \ + } \ + } \ + if (((h0).s4 == search[0]) && ((h1).s4 == search[1]) && ((h2).s4 == search[2]) && ((h3).s4 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 4); \ + } \ + } \ + \ + if (((h0).s5 == search[0]) && ((h1).s5 == search[1]) && ((h2).s5 == search[2]) && ((h3).s5 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 5); \ + } \ + } \ + \ + if (((h0).s6 == search[0]) && ((h1).s6 == search[1]) && ((h2).s6 == search[2]) && ((h3).s6 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 6); \ + } \ + } \ + \ + if (((h0).s7 == search[0]) && ((h1).s7 == search[1]) && ((h2).s7 == search[2]) && ((h3).s7 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 7); \ + } \ + } \ +} + +#define COMPARE_M_SIMD(h0,h1,h2,h3) \ +{ \ + const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 }; \ + const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 }; \ + const u32 digest_tp2[4] = { h0.s2, h1.s2, h2.s2, h3.s2 }; \ + const u32 digest_tp3[4] = { h0.s3, h1.s3, h2.s3, h3.s3 }; \ + const u32 digest_tp4[4] = { h0.s4, h1.s4, h2.s4, h3.s4 }; \ + const u32 digest_tp5[4] = { h0.s5, h1.s5, h2.s5, h3.s5 }; \ + const u32 digest_tp6[4] = { h0.s6, h1.s6, h2.s6, h3.s6 }; \ + const u32 digest_tp7[4] = { h0.s7, h1.s7, h2.s7, h3.s7 }; \ + \ + if (check (digest_tp0, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0); \ + } \ + } \ + } \ + \ + if (check (digest_tp1, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1); \ + } \ + } \ + } \ + \ + if (check (digest_tp2, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp2, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2); \ + } \ + } \ + } \ + \ + if (check (digest_tp3, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp3, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3); \ + } \ + } \ + } \ + if (check (digest_tp4, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp4, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 4); \ + } \ + } \ + } \ + \ + if (check (digest_tp5, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp5, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 5); \ + } \ + } \ + } \ + \ + if (check (digest_tp6, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp6, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 6); \ + } \ + } \ + } \ + \ + if (check (digest_tp7, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp7, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 7); \ + } \ + } \ + } \ +} + +#endif + +// vliw16 + +#if VECT_SIZE == 16 + +#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3) || ((a).s4 == (b).s4) || ((a).s5 == (b).s5) || ((a).s6 == (b).s6) || ((a).s7 == (b).s7) || ((a).s8 == (b).s8) || ((a).s9 == (b).s9) || ((a).sa == (b).sa) || ((a).sb == (b).sb) || ((a).sc == (b).sc) || ((a).sd == (b).sd) || ((a).se == (b).se) || ((a).sf == (b).sf)) +#define MATCHES_ONE_VS(a,b) (((a).s0 == (b) ) || ((a).s1 == (b) ) || ((a).s2 == (b) ) || ((a).s3 == (b) ) || ((a).s4 == (b) ) || ((a).s5 == (b) ) || ((a).s6 == (b) ) || ((a).s7 == (b) ) || ((a).s8 == (b) ) || ((a).s9 == (b) ) || ((a).sa == (b) ) || ((a).sb == (b) ) || ((a).sc == (b) ) || ((a).sd == (b) ) || ((a).se == (b) ) || ((a).sf == (b) )) + +#define COMPARE_S_SIMD(h0,h1,h2,h3) \ +{ \ + if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0); \ + } \ + } \ + \ + if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1); \ + } \ + } \ + \ + if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2); \ + } \ + } \ + \ + if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3); \ + } \ + } \ + if (((h0).s4 == search[0]) && ((h1).s4 == search[1]) && ((h2).s4 == search[2]) && ((h3).s4 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 4); \ + } \ + } \ + \ + if (((h0).s5 == search[0]) && ((h1).s5 == search[1]) && ((h2).s5 == search[2]) && ((h3).s5 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 5); \ + } \ + } \ + \ + if (((h0).s6 == search[0]) && ((h1).s6 == search[1]) && ((h2).s6 == search[2]) && ((h3).s6 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 6); \ + } \ + } \ + \ + if (((h0).s7 == search[0]) && ((h1).s7 == search[1]) && ((h2).s7 == search[2]) && ((h3).s7 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 7); \ + } \ + } \ + \ + if (((h0).s8 == search[0]) && ((h1).s8 == search[1]) && ((h2).s8 == search[2]) && ((h3).s8 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 8) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 8); \ + } \ + } \ + \ + if (((h0).s9 == search[0]) && ((h1).s9 == search[1]) && ((h2).s9 == search[2]) && ((h3).s9 == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 9) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 9); \ + } \ + } \ + \ + if (((h0).sa == search[0]) && ((h1).sa == search[1]) && ((h2).sa == search[2]) && ((h3).sa == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 10) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 10); \ + } \ + } \ + \ + if (((h0).sb == search[0]) && ((h1).sb == search[1]) && ((h2).sb == search[2]) && ((h3).sb == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 11) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 11); \ + } \ + } \ + \ + if (((h0).sc == search[0]) && ((h1).sc == search[1]) && ((h2).sc == search[2]) && ((h3).sc == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 12) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 12); \ + } \ + } \ + \ + if (((h0).sd == search[0]) && ((h1).sd == search[1]) && ((h2).sd == search[2]) && ((h3).sd == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 13) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 13); \ + } \ + } \ + \ + if (((h0).se == search[0]) && ((h1).se == search[1]) && ((h2).se == search[2]) && ((h3).se == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 14) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 14); \ + } \ + } \ + \ + if (((h0).sf == search[0]) && ((h1).sf == search[1]) && ((h2).sf == search[2]) && ((h3).sf == search[3])) \ + { \ + const u32 final_hash_pos = digests_offset + 0; \ + \ + if (vector_accessible (il_pos, il_cnt, 15) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 15); \ + } \ + } \ +} + +#define COMPARE_M_SIMD(h0,h1,h2,h3) \ +{ \ + const u32 digest_tp00[4] = { h0.s0, h1.s0, h2.s0, h3.s0 }; \ + const u32 digest_tp01[4] = { h0.s1, h1.s1, h2.s1, h3.s1 }; \ + const u32 digest_tp02[4] = { h0.s2, h1.s2, h2.s2, h3.s2 }; \ + const u32 digest_tp03[4] = { h0.s3, h1.s3, h2.s3, h3.s3 }; \ + const u32 digest_tp04[4] = { h0.s4, h1.s4, h2.s4, h3.s4 }; \ + const u32 digest_tp05[4] = { h0.s5, h1.s5, h2.s5, h3.s5 }; \ + const u32 digest_tp06[4] = { h0.s6, h1.s6, h2.s6, h3.s6 }; \ + const u32 digest_tp07[4] = { h0.s7, h1.s7, h2.s7, h3.s7 }; \ + const u32 digest_tp08[4] = { h0.s8, h1.s8, h2.s8, h3.s8 }; \ + const u32 digest_tp09[4] = { h0.s9, h1.s9, h2.s9, h3.s9 }; \ + const u32 digest_tp10[4] = { h0.sa, h1.sa, h2.sa, h3.sa }; \ + const u32 digest_tp11[4] = { h0.sb, h1.sb, h2.sb, h3.sb }; \ + const u32 digest_tp12[4] = { h0.sc, h1.sc, h2.sc, h3.sc }; \ + const u32 digest_tp13[4] = { h0.sd, h1.sd, h2.sd, h3.sd }; \ + const u32 digest_tp14[4] = { h0.se, h1.se, h2.se, h3.se }; \ + const u32 digest_tp15[4] = { h0.sf, h1.sf, h2.sf, h3.sf }; \ + \ + if (check (digest_tp00, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp00, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0); \ + } \ + } \ + } \ + \ + if (check (digest_tp01, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp01, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1); \ + } \ + } \ + } \ + \ + if (check (digest_tp02, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp02, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2); \ + } \ + } \ + } \ + \ + if (check (digest_tp03, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp03, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3); \ + } \ + } \ + } \ + \ + if (check (digest_tp04, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp04, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 4); \ + } \ + } \ + } \ + \ + if (check (digest_tp05, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp05, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 5); \ + } \ + } \ + } \ + \ + if (check (digest_tp06, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp06, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 6); \ + } \ + } \ + } \ + \ + if (check (digest_tp07, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp07, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 7); \ + } \ + } \ + } \ + \ + if (check (digest_tp08, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp08, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 8) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 8); \ + } \ + } \ + } \ + \ + if (check (digest_tp09, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp09, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 9) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 9); \ + } \ + } \ + } \ + \ + if (check (digest_tp10, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp10, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 10) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 10); \ + } \ + } \ + } \ + \ + if (check (digest_tp11, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp11, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 11) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 11); \ + } \ + } \ + } \ + \ + if (check (digest_tp12, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp12, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 12) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 12); \ + } \ + } \ + } \ + \ + if (check (digest_tp13, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp13, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 13) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 13); \ + } \ + } \ + } \ + \ + if (check (digest_tp14, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp14, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 14) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 14); \ + } \ + } \ + } \ + \ + if (check (digest_tp15, \ + bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ + bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ + bitmap_mask, \ + bitmap_shift1, \ + bitmap_shift2)) \ + { \ + int digest_pos = find_hash (digest_tp15, digests_cnt, &digests_buf[digests_offset]); \ + \ + if (digest_pos != -1) \ + { \ + const u32 final_hash_pos = digests_offset + digest_pos; \ + \ + if (vector_accessible (il_pos, il_cnt, 15) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ + { \ + mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 15); \ + } \ + } \ + } \ +} + +#endif + +#define MATCHES_NONE_VV(a,b) !(MATCHES_ONE_VV ((a), (b))) +#define MATCHES_NONE_VS(a,b) !(MATCHES_ONE_VS ((a), (b))) + +// attack-mode 0 + +inline u32x ix_create_bft (__global bf_t *bfs_buf, const u32 il_pos) +{ + #if VECT_SIZE == 1 + const u32x ix = (u32x) (bfs_buf[il_pos + 0].i); + #elif VECT_SIZE == 2 + const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i); + #elif VECT_SIZE == 4 + const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i); + #elif VECT_SIZE == 8 + const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i, bfs_buf[il_pos + 4].i, bfs_buf[il_pos + 5].i, bfs_buf[il_pos + 6].i, bfs_buf[il_pos + 7].i); + #elif VECT_SIZE == 16 + const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i, bfs_buf[il_pos + 4].i, bfs_buf[il_pos + 5].i, bfs_buf[il_pos + 6].i, bfs_buf[il_pos + 7].i, bfs_buf[il_pos + 8].i, bfs_buf[il_pos + 9].i, bfs_buf[il_pos + 10].i, bfs_buf[il_pos + 11].i, bfs_buf[il_pos + 12].i, bfs_buf[il_pos + 13].i, bfs_buf[il_pos + 14].i, bfs_buf[il_pos + 15].i); + #endif + + return ix; +} + +// attack-mode 1 + +inline u32x pwlenx_create_combt (__global comb_t *combs_buf, const u32 il_pos) +{ + #if VECT_SIZE == 1 + const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len); + #elif VECT_SIZE == 2 + const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len); + #elif VECT_SIZE == 4 + const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len); + #elif VECT_SIZE == 8 + const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len, combs_buf[il_pos + 4].pw_len, combs_buf[il_pos + 5].pw_len, combs_buf[il_pos + 6].pw_len, combs_buf[il_pos + 7].pw_len); + #elif VECT_SIZE == 16 + const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len, combs_buf[il_pos + 4].pw_len, combs_buf[il_pos + 5].pw_len, combs_buf[il_pos + 6].pw_len, combs_buf[il_pos + 7].pw_len, combs_buf[il_pos + 8].pw_len, combs_buf[il_pos + 9].pw_len, combs_buf[il_pos + 10].pw_len, combs_buf[il_pos + 11].pw_len, combs_buf[il_pos + 12].pw_len, combs_buf[il_pos + 13].pw_len, combs_buf[il_pos + 14].pw_len, combs_buf[il_pos + 15].pw_len); + #endif + + return pw_lenx; +} + +inline u32x ix_create_combt (__global comb_t *combs_buf, const u32 il_pos, const int idx) +{ + #if VECT_SIZE == 1 + const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx]); + #elif VECT_SIZE == 2 + const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx]); + #elif VECT_SIZE == 4 + const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx]); + #elif VECT_SIZE == 8 + const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx], combs_buf[il_pos + 4].i[idx], combs_buf[il_pos + 5].i[idx], combs_buf[il_pos + 6].i[idx], combs_buf[il_pos + 7].i[idx]); + #elif VECT_SIZE == 16 + const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx], combs_buf[il_pos + 4].i[idx], combs_buf[il_pos + 5].i[idx], combs_buf[il_pos + 6].i[idx], combs_buf[il_pos + 7].i[idx], combs_buf[il_pos + 8].i[idx], combs_buf[il_pos + 9].i[idx], combs_buf[il_pos + 10].i[idx], combs_buf[il_pos + 11].i[idx], combs_buf[il_pos + 12].i[idx], combs_buf[il_pos + 13].i[idx], combs_buf[il_pos + 14].i[idx], combs_buf[il_pos + 15].i[idx]); + #endif + + return ix; +} + +#if VECT_SIZE == 1 +#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 1) + 0].var[(idx)]) +#elif VECT_SIZE == 2 +#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 2) + 0].var[(idx)], (arr)[((gid) * 2) + 1].var[(idx)]) +#elif VECT_SIZE == 4 +#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 4) + 0].var[(idx)], (arr)[((gid) * 4) + 1].var[(idx)], (arr)[((gid) * 4) + 2].var[(idx)], (arr)[((gid) * 4) + 3].var[(idx)]) +#elif VECT_SIZE == 8 +#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 8) + 0].var[(idx)], (arr)[((gid) * 8) + 1].var[(idx)], (arr)[((gid) * 8) + 2].var[(idx)], (arr)[((gid) * 8) + 3].var[(idx)], (arr)[((gid) * 8) + 4].var[(idx)], (arr)[((gid) * 8) + 5].var[(idx)], (arr)[((gid) * 8) + 6].var[(idx)], (arr)[((gid) * 8) + 7].var[(idx)]) +#elif VECT_SIZE == 16 +#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 16) + 0].var[(idx)], (arr)[((gid) * 16) + 1].var[(idx)], (arr)[((gid) * 16) + 2].var[(idx)], (arr)[((gid) * 16) + 3].var[(idx)], (arr)[((gid) * 16) + 4].var[(idx)], (arr)[((gid) * 16) + 5].var[(idx)], (arr)[((gid) * 16) + 6].var[(idx)], (arr)[((gid) * 16) + 7].var[(idx)], (arr)[((gid) * 16) + 8].var[(idx)], (arr)[((gid) * 16) + 9].var[(idx)], (arr)[((gid) * 16) + 10].var[(idx)], (arr)[((gid) * 16) + 11].var[(idx)], (arr)[((gid) * 16) + 12].var[(idx)], (arr)[((gid) * 16) + 13].var[(idx)], (arr)[((gid) * 16) + 14].var[(idx)], (arr)[((gid) * 16) + 15].var[(idx)]) +#endif + +#if VECT_SIZE == 1 +#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 1) + 0].var) +#elif VECT_SIZE == 2 +#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 2) + 0].var, (arr)[((gid) * 2) + 1].var) +#elif VECT_SIZE == 4 +#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 4) + 0].var, (arr)[((gid) * 4) + 1].var, (arr)[((gid) * 4) + 2].var, (arr)[((gid) * 4) + 3].var) +#elif VECT_SIZE == 8 +#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 8) + 0].var, (arr)[((gid) * 8) + 1].var, (arr)[((gid) * 8) + 2].var, (arr)[((gid) * 8) + 3].var, (arr)[((gid) * 8) + 4].var, (arr)[((gid) * 8) + 5].var, (arr)[((gid) * 8) + 6].var, (arr)[((gid) * 8) + 7].var) +#elif VECT_SIZE == 16 +#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 16) + 0].var, (arr)[((gid) * 16) + 1].var, (arr)[((gid) * 16) + 2].var, (arr)[((gid) * 16) + 3].var, (arr)[((gid) * 16) + 4].var, (arr)[((gid) * 16) + 5].var, (arr)[((gid) * 16) + 6].var, (arr)[((gid) * 16) + 7].var, (arr)[((gid) * 16) + 8].var, (arr)[((gid) * 16) + 9].var, (arr)[((gid) * 16) + 10].var, (arr)[((gid) * 16) + 11].var, (arr)[((gid) * 16) + 12].var, (arr)[((gid) * 16) + 13].var, (arr)[((gid) * 16) + 14].var, (arr)[((gid) * 16) + 15].var) +#endif + +#if VECT_SIZE == 1 +#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 1) + 0].var[(idx)]) +#elif VECT_SIZE == 2 +#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 2) + 0].var[(idx)], (arr)[((gid) * 2) + 1].var[(idx)]) +#elif VECT_SIZE == 4 +#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 4) + 0].var[(idx)], (arr)[((gid) * 4) + 1].var[(idx)], (arr)[((gid) * 4) + 2].var[(idx)], (arr)[((gid) * 4) + 3].var[(idx)]) +#elif VECT_SIZE == 8 +#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 8) + 0].var[(idx)], (arr)[((gid) * 8) + 1].var[(idx)], (arr)[((gid) * 8) + 2].var[(idx)], (arr)[((gid) * 8) + 3].var[(idx)], (arr)[((gid) * 8) + 4].var[(idx)], (arr)[((gid) * 8) + 5].var[(idx)], (arr)[((gid) * 8) + 6].var[(idx)], (arr)[((gid) * 8) + 7].var[(idx)]) +#elif VECT_SIZE == 16 +#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 16) + 0].var[(idx)], (arr)[((gid) * 16) + 1].var[(idx)], (arr)[((gid) * 16) + 2].var[(idx)], (arr)[((gid) * 16) + 3].var[(idx)], (arr)[((gid) * 16) + 4].var[(idx)], (arr)[((gid) * 16) + 5].var[(idx)], (arr)[((gid) * 16) + 6].var[(idx)], (arr)[((gid) * 16) + 7].var[(idx)], (arr)[((gid) * 16) + 8].var[(idx)], (arr)[((gid) * 16) + 9].var[(idx)], (arr)[((gid) * 16) + 10].var[(idx)], (arr)[((gid) * 16) + 11].var[(idx)], (arr)[((gid) * 16) + 12].var[(idx)], (arr)[((gid) * 16) + 13].var[(idx)], (arr)[((gid) * 16) + 14].var[(idx)], (arr)[((gid) * 16) + 15].var[(idx)]) +#endif + +#if VECT_SIZE == 1 +#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 1) + 0].var) +#elif VECT_SIZE == 2 +#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 2) + 0].var, (arr)[((gid) * 2) + 1].var) +#elif VECT_SIZE == 4 +#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 4) + 0].var, (arr)[((gid) * 4) + 1].var, (arr)[((gid) * 4) + 2].var, (arr)[((gid) * 4) + 3].var) +#elif VECT_SIZE == 8 +#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 8) + 0].var, (arr)[((gid) * 8) + 1].var, (arr)[((gid) * 8) + 2].var, (arr)[((gid) * 8) + 3].var, (arr)[((gid) * 8) + 4].var, (arr)[((gid) * 8) + 5].var, (arr)[((gid) * 8) + 6].var, (arr)[((gid) * 8) + 7].var) +#elif VECT_SIZE == 16 +#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 16) + 0].var, (arr)[((gid) * 16) + 1].var, (arr)[((gid) * 16) + 2].var, (arr)[((gid) * 16) + 3].var, (arr)[((gid) * 16) + 4].var, (arr)[((gid) * 16) + 5].var, (arr)[((gid) * 16) + 6].var, (arr)[((gid) * 16) + 7].var, (arr)[((gid) * 16) + 8].var, (arr)[((gid) * 16) + 9].var, (arr)[((gid) * 16) + 10].var, (arr)[((gid) * 16) + 11].var, (arr)[((gid) * 16) + 12].var, (arr)[((gid) * 16) + 13].var, (arr)[((gid) * 16) + 14].var, (arr)[((gid) * 16) + 15].var) +#endif + +#if VECT_SIZE == 1 +#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 1) + 0].var[(idx)] = val; +#elif VECT_SIZE == 2 +#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 2) + 0].var[(idx)] = val.s0; (arr)[((gid) * 2) + 1].var[(idx)] = val.s1; +#elif VECT_SIZE == 4 +#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 4) + 0].var[(idx)] = val.s0; (arr)[((gid) * 4) + 1].var[(idx)] = val.s1; (arr)[((gid) * 4) + 2].var[(idx)] = val.s2; (arr)[((gid) * 4) + 3].var[(idx)] = val.s3; +#elif VECT_SIZE == 8 +#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 8) + 0].var[(idx)] = val.s0; (arr)[((gid) * 8) + 1].var[(idx)] = val.s1; (arr)[((gid) * 8) + 2].var[(idx)] = val.s2; (arr)[((gid) * 8) + 3].var[(idx)] = val.s3; (arr)[((gid) * 8) + 4].var[(idx)] = val.s4; (arr)[((gid) * 8) + 5].var[(idx)] = val.s5; (arr)[((gid) * 8) + 6].var[(idx)] = val.s6; (arr)[((gid) * 8) + 7].var[(idx)] = val.s7; +#elif VECT_SIZE == 16 +#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 16) + 0].var[(idx)] = val.s0; (arr)[((gid) * 16) + 1].var[(idx)] = val.s1; (arr)[((gid) * 16) + 2].var[(idx)] = val.s2; (arr)[((gid) * 16) + 3].var[(idx)] = val.s3; (arr)[((gid) * 16) + 4].var[(idx)] = val.s4; (arr)[((gid) * 16) + 5].var[(idx)] = val.s5; (arr)[((gid) * 16) + 6].var[(idx)] = val.s6; (arr)[((gid) * 16) + 7].var[(idx)] = val.s7; (arr)[((gid) * 16) + 8].var[(idx)] = val.s8; (arr)[((gid) * 16) + 9].var[(idx)] = val.s9; (arr)[((gid) * 16) + 10].var[(idx)] = val.sa; (arr)[((gid) * 16) + 11].var[(idx)] = val.sb; (arr)[((gid) * 16) + 12].var[(idx)] = val.sc; (arr)[((gid) * 16) + 13].var[(idx)] = val.sd; (arr)[((gid) * 16) + 14].var[(idx)] = val.se; (arr)[((gid) * 16) + 15].var[(idx)] = val.sf; +#endif diff --git a/OpenCL/inc_types.cl b/OpenCL/inc_types.cl new file mode 100644 index 0000000..1d3a7b8 --- /dev/null +++ b/OpenCL/inc_types.cl @@ -0,0 +1,1480 @@ +/** + * Authors.....: Jens Steube + * magnum + * + * License.....: MIT + */ + +typedef uchar u8; +typedef ushort u16; +typedef uint u32; +typedef ulong u64; + +#ifndef NEW_SIMD_CODE +#undef VECT_SIZE +#define VECT_SIZE 1 +#endif + +#define CONCAT(a, b) a##b +#define VTYPE(type, width) CONCAT(type, width) + +#if VECT_SIZE == 1 +typedef uchar u8x; +typedef ushort u16x; +typedef uint u32x; +typedef ulong u64x; +#else +typedef VTYPE(uchar, VECT_SIZE) u8x; +typedef VTYPE(ushort, VECT_SIZE) u16x; +typedef VTYPE(uint, VECT_SIZE) u32x; +typedef VTYPE(ulong, VECT_SIZE) u64x; +#endif + +inline u32 l32_from_64_S (u64 a) +{ + const u32 r = (u32) (a); + + return r; +} + +inline u32 h32_from_64_S (u64 a) +{ + a >>= 32; + + const u32 r = (u32) (a); + + return r; +} + +inline u64 hl32_to_64_S (const u32 a, const u32 b) +{ + return as_ulong ((uint2) (b, a)); +} + +inline u32x l32_from_64 (u64x a) +{ + u32x r; + + #if VECT_SIZE == 1 + r = (u32) a; + #endif + + #if VECT_SIZE >= 2 + r.s0 = (u32) a.s0; + r.s1 = (u32) a.s1; + #endif + + #if VECT_SIZE >= 4 + r.s2 = (u32) a.s2; + r.s3 = (u32) a.s3; + #endif + + #if VECT_SIZE >= 8 + r.s4 = (u32) a.s4; + r.s5 = (u32) a.s5; + r.s6 = (u32) a.s6; + r.s7 = (u32) a.s7; + #endif + + #if VECT_SIZE >= 16 + r.s8 = (u32) a.s8; + r.s9 = (u32) a.s9; + r.sa = (u32) a.sa; + r.sb = (u32) a.sb; + r.sc = (u32) a.sc; + r.sd = (u32) a.sd; + r.se = (u32) a.se; + r.sf = (u32) a.sf; + #endif + + return r; +} + +inline u32x h32_from_64 (u64x a) +{ + a >>= 32; + + u32x r; + + #if VECT_SIZE == 1 + r = (u32) a; + #endif + + #if VECT_SIZE >= 2 + r.s0 = (u32) a.s0; + r.s1 = (u32) a.s1; + #endif + + #if VECT_SIZE >= 4 + r.s2 = (u32) a.s2; + r.s3 = (u32) a.s3; + #endif + + #if VECT_SIZE >= 8 + r.s4 = (u32) a.s4; + r.s5 = (u32) a.s5; + r.s6 = (u32) a.s6; + r.s7 = (u32) a.s7; + #endif + + #if VECT_SIZE >= 16 + r.s8 = (u32) a.s8; + r.s9 = (u32) a.s9; + r.sa = (u32) a.sa; + r.sb = (u32) a.sb; + r.sc = (u32) a.sc; + r.sd = (u32) a.sd; + r.se = (u32) a.se; + r.sf = (u32) a.sf; + #endif + + return r; +} + +inline u64x hl32_to_64 (const u32x a, const u32x b) +{ + u64x r; + + #if VECT_SIZE == 1 + r = as_ulong ((uint2) (b, a)); + #endif + + #if VECT_SIZE >= 2 + r.s0 = as_ulong ((uint2) (b.s0, a.s0)); + r.s1 = as_ulong ((uint2) (b.s1, a.s1)); + #endif + + #if VECT_SIZE >= 4 + r.s2 = as_ulong ((uint2) (b.s2, a.s2)); + r.s3 = as_ulong ((uint2) (b.s3, a.s3)); + #endif + + #if VECT_SIZE >= 8 + r.s4 = as_ulong ((uint2) (b.s4, a.s4)); + r.s5 = as_ulong ((uint2) (b.s5, a.s5)); + r.s6 = as_ulong ((uint2) (b.s6, a.s6)); + r.s7 = as_ulong ((uint2) (b.s7, a.s7)); + #endif + + #if VECT_SIZE >= 16 + r.s8 = as_ulong ((uint2) (b.s8, a.s8)); + r.s9 = as_ulong ((uint2) (b.s9, a.s9)); + r.sa = as_ulong ((uint2) (b.sa, a.sa)); + r.sb = as_ulong ((uint2) (b.sb, a.sb)); + r.sc = as_ulong ((uint2) (b.sc, a.sc)); + r.sd = as_ulong ((uint2) (b.sd, a.sd)); + r.se = as_ulong ((uint2) (b.se, a.se)); + r.sf = as_ulong ((uint2) (b.sf, a.sf)); + #endif + + return r; +} + +#ifdef IS_AMD +inline u32 swap32_S (const u32 v) +{ + return (as_uint (as_uchar4 (v).s3210)); +} + +inline u64 swap64_S (const u64 v) +{ + return (as_ulong (as_uchar8 (v).s76543210)); +} + +inline u32 rotr32_S (const u32 a, const u32 n) +{ + return rotate (a, 32 - n); +} + +inline u32 rotl32_S (const u32 a, const u32 n) +{ + return rotate (a, n); +} + +inline u64 rotr64_S (const u64 a, const u32 n) +{ + const u32 a0 = h32_from_64_S (a); + const u32 a1 = l32_from_64_S (a); + + const u32 t0 = (n >= 32) ? amd_bitalign (a0, a1, n - 32) : amd_bitalign (a1, a0, n); + const u32 t1 = (n >= 32) ? amd_bitalign (a1, a0, n - 32) : amd_bitalign (a0, a1, n); + + const u64 r = hl32_to_64_S (t0, t1); + + return r; +} + +inline u64 rotl64_S (const u64 a, const u32 n) +{ + return rotr64_S (a, 64 - n); +} + +inline u32x swap32 (const u32x v) +{ + return ((v >> 24) & 0x000000ff) + | ((v >> 8) & 0x0000ff00) + | ((v << 8) & 0x00ff0000) + | ((v << 24) & 0xff000000); +} + +inline u64x swap64 (const u64x v) +{ + return ((v >> 56) & 0x00000000000000ff) + | ((v >> 40) & 0x000000000000ff00) + | ((v >> 24) & 0x0000000000ff0000) + | ((v >> 8) & 0x00000000ff000000) + | ((v << 8) & 0x000000ff00000000) + | ((v << 24) & 0x0000ff0000000000) + | ((v << 40) & 0x00ff000000000000) + | ((v << 56) & 0xff00000000000000); +} + +inline u32x rotr32 (const u32x a, const u32 n) +{ + return rotate (a, 32 - n); +} + +inline u32x rotl32 (const u32x a, const u32 n) +{ + return rotate (a, n); +} + +inline u64x rotr64 (const u64x a, const u32 n) +{ + const u32x a0 = h32_from_64 (a); + const u32x a1 = l32_from_64 (a); + + const u32x t0 = (n >= 32) ? amd_bitalign (a0, a1, n - 32) : amd_bitalign (a1, a0, n); + const u32x t1 = (n >= 32) ? amd_bitalign (a1, a0, n - 32) : amd_bitalign (a0, a1, n); + + const u64x r = hl32_to_64 (t0, t1); + + return r; +} + +inline u64x rotl64 (const u64x a, const u32 n) +{ + return rotr64 (a, 64 - n); +} + +inline u32x __bfe (const u32x a, const u32x b, const u32x c) +{ + return amd_bfe (a, b, c); +} + +inline u32 __bfe_S (const u32 a, const u32 b, const u32 c) +{ + return amd_bfe (a, b, c); +} + +inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c) +{ + return amd_bytealign (a, b, c); +} +#endif + +#ifdef IS_NV +inline u32 swap32_S (const u32 v) +{ + u32 r; + + asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(r) : "r"(v)); + + return r; +} + +inline u64 swap64_S (const u64 v) +{ + u32 il; + u32 ir; + + asm ("mov.b64 {%0, %1}, %2;" : "=r"(il), "=r"(ir) : "l"(v)); + + u32 tl; + u32 tr; + + asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(tl) : "r"(il)); + asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(tr) : "r"(ir)); + + u64 r; + + asm ("mov.b64 %0, {%1, %2};" : "=l"(r) : "r"(tr), "r"(tl)); + + return r; +} + +inline u32 rotr32_S (const u32 a, const u32 n) +{ + return rotate (a, 32 - n); +} + +inline u32 rotl32_S (const u32 a, const u32 n) +{ + return rotate (a, n); +} + +inline u64 rotr64_S (const u64 a, const u32 n) +{ + return rotate (a, (u64) 64 - n); +} + +inline u64 rotl64_S (const u64 a, const u32 n) +{ + return rotr64_S (a, 64 - n); +} + +inline u32x swap32 (const u32x v) +{ + return ((v >> 24) & 0x000000ff) + | ((v >> 8) & 0x0000ff00) + | ((v << 8) & 0x00ff0000) + | ((v << 24) & 0xff000000); +} + +inline u64x swap64 (const u64x v) +{ + return ((v >> 56) & 0x00000000000000ff) + | ((v >> 40) & 0x000000000000ff00) + | ((v >> 24) & 0x0000000000ff0000) + | ((v >> 8) & 0x00000000ff000000) + | ((v << 8) & 0x000000ff00000000) + | ((v << 24) & 0x0000ff0000000000) + | ((v << 40) & 0x00ff000000000000) + | ((v << 56) & 0xff00000000000000); +} + +inline u32x rotr32 (const u32x a, const u32 n) +{ + return rotate (a, 32 - n); +} + +inline u32x rotl32 (const u32x a, const u32 n) +{ + return rotate (a, n); +} + +inline u64x rotr64 (const u64x a, const u32 n) +{ + return rotate (a, (u64) 64 - n); +} + +inline u64x rotl64 (const u64x a, const u32 n) +{ + return rotate (a, (u64) n); +} + +inline u32x __byte_perm (const u32x a, const u32x b, const u32x c) +{ + u32x r; + + #if VECT_SIZE == 1 + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c) ); + #endif + + #if VECT_SIZE >= 2 + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(a.s0), "r"(b.s0), "r"(c.s0)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(a.s1), "r"(b.s1), "r"(c.s1)); + #endif + + #if VECT_SIZE >= 4 + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(a.s2), "r"(b.s2), "r"(c.s2)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(a.s3), "r"(b.s3), "r"(c.s3)); + #endif + + #if VECT_SIZE >= 8 + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(a.s4), "r"(b.s4), "r"(c.s4)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(a.s5), "r"(b.s5), "r"(c.s5)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(a.s6), "r"(b.s6), "r"(c.s6)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(a.s7), "r"(b.s7), "r"(c.s7)); + #endif + + #if VECT_SIZE >= 16 + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(a.s8), "r"(b.s8), "r"(c.s8)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(a.s9), "r"(b.s9), "r"(c.s9)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(a.sa), "r"(b.sa), "r"(c.sa)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(a.sb), "r"(b.sb), "r"(c.sb)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(a.sc), "r"(b.sc), "r"(c.sc)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(a.sd), "r"(b.sd), "r"(c.sd)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(a.se), "r"(b.se), "r"(c.se)); + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(a.sf), "r"(b.sf), "r"(c.sf)); + #endif + + return r; +} + +inline u32 __byte_perm_S (const u32 a, const u32 b, const u32 c) +{ + u32 r; + + asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + + return r; +} + +inline u32x __bfe (const u32x a, const u32x b, const u32x c) +{ + u32x r; + + #if VECT_SIZE == 1 + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + #endif + + #if VECT_SIZE >= 2 + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(a.s0), "r"(b.s0), "r"(c.s0)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(a.s1), "r"(b.s1), "r"(c.s1)); + #endif + + #if VECT_SIZE >= 4 + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(a.s2), "r"(b.s2), "r"(c.s2)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(a.s3), "r"(b.s3), "r"(c.s3)); + #endif + + #if VECT_SIZE >= 8 + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(a.s4), "r"(b.s4), "r"(c.s4)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(a.s5), "r"(b.s5), "r"(c.s5)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(a.s6), "r"(b.s6), "r"(c.s6)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(a.s7), "r"(b.s7), "r"(c.s7)); + #endif + + #if VECT_SIZE >= 16 + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(a.s8), "r"(b.s8), "r"(c.s8)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(a.s9), "r"(b.s9), "r"(c.s9)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(a.sa), "r"(b.sa), "r"(c.sa)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(a.sb), "r"(b.sb), "r"(c.sb)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(a.sc), "r"(b.sc), "r"(c.sc)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(a.sd), "r"(b.sd), "r"(c.sd)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(a.se), "r"(b.se), "r"(c.se)); + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(a.sf), "r"(b.sf), "r"(c.sf)); + #endif + + return r; +} + +inline u32 __bfe_S (const u32 a, const u32 b, const u32 c) +{ + u32 r; + + asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + + return r; +} + +inline u32x amd_bytealign (const u32x a, const u32x b, const u32x c) +{ + u32x r; + + #if CUDA_ARCH >= 350 + + #if VECT_SIZE == 1 + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(b), "r"(a), "r"((c & 3) * 8)); + #endif + + #if VECT_SIZE >= 2 + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(b.s0), "r"(a.s0), "r"((c.s0 & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(b.s1), "r"(a.s1), "r"((c.s1 & 3) * 8)); + #endif + + #if VECT_SIZE >= 4 + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(b.s2), "r"(a.s2), "r"((c.s2 & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(b.s3), "r"(a.s3), "r"((c.s3 & 3) * 8)); + #endif + + #if VECT_SIZE >= 8 + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(b.s4), "r"(a.s4), "r"((c.s4 & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(b.s5), "r"(a.s5), "r"((c.s5 & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(b.s6), "r"(a.s6), "r"((c.s6 & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(b.s7), "r"(a.s7), "r"((c.s7 & 3) * 8)); + #endif + + #if VECT_SIZE >= 16 + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(b.s8), "r"(a.s8), "r"((c.s8 & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(b.s9), "r"(a.s9), "r"((c.s9 & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(b.sa), "r"(a.sa), "r"((c.sa & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(b.sb), "r"(a.sb), "r"((c.sb & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(b.sc), "r"(a.sc), "r"((c.sc & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(b.sd), "r"(a.sd), "r"((c.sd & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(b.se), "r"(a.se), "r"((c.se & 3) * 8)); + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(b.sf), "r"(a.sf), "r"((c.sf & 3) * 8)); + #endif + + #else + + r = __byte_perm (b, a, ((u32x) (0x76543210) >> ((c & 3) * 4)) & 0xffff); + + #endif + + return r; +} + +inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c) +{ + u32 r; + + #if CUDA_ARCH >= 350 + + asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(b), "r"(a), "r"((c & 3) * 8)); + + #else + + r = __byte_perm_S (b, a, (0x76543210 >> ((c & 3) * 4)) & 0xffff); + + #endif + + return r; +} +#endif + +#ifdef IS_GENERIC +inline u32 swap32_S (const u32 v) +{ + return (as_uint (as_uchar4 (v).s3210)); +} + +inline u64 swap64_S (const u64 v) +{ + return (as_ulong (as_uchar8 (v).s76543210)); +} + +inline u32 rotr32_S (const u32 a, const u32 n) +{ + return rotate (a, 32 - n); +} + +inline u32 rotl32_S (const u32 a, const u32 n) +{ + return rotate (a, n); +} + +inline u64 rotr64_S (const u64 a, const u32 n) +{ + return rotate (a, (u64) 64 - n); +} + +inline u64 rotl64_S (const u64 a, const u32 n) +{ + return rotate (a, (u64) n); +} + +inline u32x swap32 (const u32x v) +{ + return ((v >> 24) & 0x000000ff) + | ((v >> 8) & 0x0000ff00) + | ((v << 8) & 0x00ff0000) + | ((v << 24) & 0xff000000); +} + +inline u64x swap64 (const u64x v) +{ + return ((v >> 56) & 0x00000000000000ff) + | ((v >> 40) & 0x000000000000ff00) + | ((v >> 24) & 0x0000000000ff0000) + | ((v >> 8) & 0x00000000ff000000) + | ((v << 8) & 0x000000ff00000000) + | ((v << 24) & 0x0000ff0000000000) + | ((v << 40) & 0x00ff000000000000) + | ((v << 56) & 0xff00000000000000); +} + +inline u32x rotr32 (const u32x a, const u32 n) +{ + return rotate (a, 32 - n); +} + +inline u32x rotl32 (const u32x a, const u32 n) +{ + return rotate (a, n); +} + +inline u64x rotr64 (const u64x a, const u32 n) +{ + return rotate (a, (u64) 64 - n); +} + +inline u64x rotl64 (const u64x a, const u32 n) +{ + return rotate (a, (u64) n); +} + +inline u32x __bfe (const u32x a, const u32x b, const u32x c) +{ + #define BIT(x) ((u32x) (1u) << (x)) + #define BIT_MASK(x) (BIT (x) - 1) + #define BFE(x,y,z) (((x) >> (y)) & BIT_MASK (z)) + + return BFE (a, b, c); + + #undef BIT + #undef BIT_MASK + #undef BFE +} + +inline u32 __bfe_S (const u32 a, const u32 b, const u32 c) +{ + #define BIT(x) (1u << (x)) + #define BIT_MASK(x) (BIT (x) - 1) + #define BFE(x,y,z) (((x) >> (y)) & BIT_MASK (z)) + + return BFE (a, b, c); + + #undef BIT + #undef BIT_MASK + #undef BFE +} + +inline u32x amd_bytealign (const u32x a, const u32x b, const u32 c) +{ + #if VECT_SIZE == 1 + const u64x tmp = ((((u64x) (a)) << 32) | ((u64x) (b))) >> ((c & 3) * 8); + + return (u32x) (tmp); + #endif + + #if VECT_SIZE == 2 + const u64x tmp = ((((u64x) (a.s0, a.s1)) << 32) | ((u64x) (b.s0, b.s1))) >> ((c & 3) * 8); + + return (u32x) (tmp.s0, tmp.s1); + #endif + + #if VECT_SIZE == 4 + const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3))) >> ((c & 3) * 8); + + return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3); + #endif + + #if VECT_SIZE == 8 + const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3, a.s4, a.s5, a.s6, a.s7)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3, b.s4, b.s5, b.s6, b.s7))) >> ((c & 3) * 8); + + return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3, tmp.s4, tmp.s5, tmp.s6, tmp.s7); + #endif + + #if VECT_SIZE == 16 + const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3, a.s4, a.s5, a.s6, a.s7, a.s8, a.s9, a.sa, a.sb, a.sc, a.sd, a.se, a.sf)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3, b.s4, b.s5, b.s6, b.s7, b.s8, b.s9, b.sa, b.sb, b.sc, b.sd, b.se, b.sf))) >> ((c & 3) * 8); + + return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3, tmp.s4, tmp.s5, tmp.s6, tmp.s7, tmp.s8, tmp.s9, tmp.sa, tmp.sb, tmp.sc, tmp.sd, tmp.se, tmp.sf); + #endif +} + +inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c) +{ + const u64 tmp = ((((u64) a) << 32) | ((u64) b)) >> ((c & 3) * 8); + + return (u32) (tmp); +} + +#endif + +typedef struct +{ + #if defined _DES_ + u32 digest_buf[4]; + #elif defined _MD4_ + u32 digest_buf[4]; + #elif defined _MD5_ + u32 digest_buf[4]; + #elif defined _MD5H_ + u32 digest_buf[4]; + #elif defined _SHA1_ + u32 digest_buf[5]; + #elif defined _BCRYPT_ + u32 digest_buf[6]; + #elif defined _SHA256_ + u32 digest_buf[8]; + #elif defined _SHA384_ + u32 digest_buf[16]; + #elif defined _SHA512_ + u32 digest_buf[16]; + #elif defined _KECCAK_ + u32 digest_buf[50]; + #elif defined _RIPEMD160_ + u32 digest_buf[5]; + #elif defined _WHIRLPOOL_ + u32 digest_buf[16]; + #elif defined _GOST_ + u32 digest_buf[8]; + #elif defined _GOST2012_256_ + u32 digest_buf[8]; + #elif defined _GOST2012_512_ + u32 digest_buf[16]; + #elif defined _SAPB_ + u32 digest_buf[4]; + #elif defined _SAPG_ + u32 digest_buf[5]; + #elif defined _MYSQL323_ + u32 digest_buf[4]; + #elif defined _LOTUS5_ + u32 digest_buf[4]; + #elif defined _LOTUS6_ + u32 digest_buf[4]; + #elif defined _SCRYPT_ + u32 digest_buf[8]; + #elif defined _LOTUS8_ + u32 digest_buf[4]; + #elif defined _OFFICE2007_ + u32 digest_buf[4]; + #elif defined _OFFICE2010_ + u32 digest_buf[4]; + #elif defined _OFFICE2013_ + u32 digest_buf[4]; + #elif defined _OLDOFFICE01_ + u32 digest_buf[4]; + #elif defined _OLDOFFICE34_ + u32 digest_buf[4]; + #elif defined _SIPHASH_ + u32 digest_buf[4]; + #elif defined _PBKDF2_MD5_ + u32 digest_buf[32]; + #elif defined _PBKDF2_SHA1_ + u32 digest_buf[32]; + #elif defined _PBKDF2_SHA256_ + u32 digest_buf[32]; + #elif defined _PBKDF2_SHA512_ + u32 digest_buf[32]; + #elif defined _PDF17L8_ + u32 digest_buf[8]; + #elif defined _CRC32_ + u32 digest_buf[4]; + #elif defined _SEVEN_ZIP_ + u32 digest_buf[4]; + #elif defined _ANDROIDFDE_ + u32 digest_buf[4]; + #elif defined _DCC2_ + u32 digest_buf[4]; + #elif defined _WPA_ + u32 digest_buf[4]; + #elif defined _MD5_SHA1_ + u32 digest_buf[4]; + #elif defined _SHA1_MD5_ + u32 digest_buf[5]; + #elif defined _NETNTLMV2_ + u32 digest_buf[4]; + #elif defined _KRB5PA_ + u32 digest_buf[4]; + #elif defined _CLOUDKEY_ + u32 digest_buf[8]; + #elif defined _SCRYPT_ + u32 digest_buf[4]; + #elif defined _PSAFE2_ + u32 digest_buf[5]; + #elif defined _LOTUS8_ + u32 digest_buf[4]; + #elif defined _RAR3_ + u32 digest_buf[4]; + #elif defined _SHA256_SHA1_ + u32 digest_buf[8]; + #elif defined _MS_DRSR_ + u32 digest_buf[8]; + #elif defined _ANDROIDFDE_SAMSUNG_ + u32 digest_buf[8]; + #elif defined _RAR5_ + u32 digest_buf[4]; + #elif defined _KRB5TGS_ + u32 digest_buf[4]; + #elif defined _AXCRYPT_ + u32 digest_buf[4]; + #elif defined _KEEPASS_ + u32 digest_buf[4]; + #elif defined _ZIP2_ + u32 digest_buf[4]; + #endif + +} digest_t; + +typedef struct +{ + u32 salt_buf[16]; + u32 salt_buf_pc[8]; + + u32 salt_len; + u32 salt_iter; + u32 salt_sign[2]; + + u32 keccak_mdlen; + u32 truecrypt_mdlen; + + u32 digests_cnt; + u32 digests_done; + + u32 digests_offset; + + u32 scrypt_N; + u32 scrypt_r; + u32 scrypt_p; + u32 scrypt_tmto; + u32 scrypt_phy; + +} salt_t; + +typedef struct +{ + int V; + int R; + int P; + + int enc_md; + + u32 id_buf[8]; + u32 u_buf[32]; + u32 o_buf[32]; + + int id_len; + int o_len; + int u_len; + + u32 rc4key[2]; + u32 rc4data[2]; + +} pdf_t; + +typedef struct +{ + u32 pke[25]; + u32 eapol[64]; + int eapol_size; + int keyver; + u8 orig_mac1[6]; + u8 orig_mac2[6]; + u8 orig_nonce1[32]; + u8 orig_nonce2[32]; + +} wpa_t; + +typedef struct +{ + u32 cry_master_buf[64]; + u32 ckey_buf[64]; + u32 public_key_buf[64]; + + u32 cry_master_len; + u32 ckey_len; + u32 public_key_len; + +} bitcoin_wallet_t; + +typedef struct +{ + u32 salt_buf[30]; + u32 salt_len; + + u32 esalt_buf[38]; + u32 esalt_len; + +} sip_t; + +typedef struct +{ + u32 data[384]; + +} androidfde_t; + +typedef struct +{ + u32 nr_buf[16]; + u32 nr_len; + + u32 msg_buf[128]; + u32 msg_len; + +} ikepsk_t; + +typedef struct +{ + u32 user_len; + u32 domain_len; + u32 srvchall_len; + u32 clichall_len; + + u32 userdomain_buf[64]; + u32 chall_buf[256]; + +} netntlm_t; + +typedef struct +{ + u32 user[16]; + u32 realm[16]; + u32 salt[32]; + u32 timestamp[16]; + u32 checksum[4]; + +} krb5pa_t; + +typedef struct +{ + u32 account_info[512]; + u32 checksum[4]; + u32 edata2[2560]; + u32 edata2_len; + +} krb5tgs_t; + +typedef struct +{ + u32 salt_buf[16]; + u32 data_buf[112]; + u32 keyfile_buf[16]; + u32 signature; + +} tc_t; + +typedef struct +{ + u32 salt_buf[16]; + +} pbkdf2_md5_t; + +typedef struct +{ + u32 salt_buf[16]; + +} pbkdf2_sha1_t; + +typedef struct +{ + u32 salt_buf[16]; + +} pbkdf2_sha256_t; + +typedef struct +{ + u32 salt_buf[32]; + +} pbkdf2_sha512_t; + +typedef struct +{ + u32 salt_buf[128]; + u32 salt_len; + +} rakp_t; + +typedef struct +{ + u32 data_len; + u32 data_buf[512]; + +} cloudkey_t; + +typedef struct +{ + u32 encryptedVerifier[4]; + u32 encryptedVerifierHash[5]; + + u32 keySize; + +} office2007_t; + +typedef struct +{ + u32 encryptedVerifier[4]; + u32 encryptedVerifierHash[8]; + +} office2010_t; + +typedef struct +{ + u32 encryptedVerifier[4]; + u32 encryptedVerifierHash[8]; + +} office2013_t; + +typedef struct +{ + u32 version; + u32 encryptedVerifier[4]; + u32 encryptedVerifierHash[4]; + u32 rc4key[2]; + +} oldoffice01_t; + +typedef struct +{ + u32 version; + u32 encryptedVerifier[4]; + u32 encryptedVerifierHash[5]; + u32 rc4key[2]; + +} oldoffice34_t; + +typedef struct +{ + u32 salt_buf[128]; + u32 salt_len; + + u32 pc_digest[5]; + u32 pc_offset; + +} pstoken_t; + +typedef struct +{ + u32 type; + u32 mode; + u32 magic; + u32 salt_len; + u32 salt_buf[4]; + u32 verify_bytes; + u32 compress_length; + u32 data_len; + u32 data_buf[2048]; + u32 auth_len; + u32 auth_buf[4]; + +} zip2_t; + +typedef struct +{ + u32 version; + u32 algorithm; + + /* key-file handling */ + u32 keyfile_len; + u32 keyfile[8]; + + u32 final_random_seed[8]; + u32 transf_random_seed[8]; + u32 enc_iv[4]; + u32 contents_hash[8]; + + /* specific to version 1 */ + u32 contents_len; + u32 contents[75000]; + + /* specific to version 2 */ + u32 expected_bytes[8]; + +} keepass_t; + +typedef struct +{ + u32 digest[4]; + u32 out[4]; + +} pdf14_tmp_t; + +typedef struct +{ + union + { + u32 dgst32[16]; + u64 dgst64[8]; + }; + + u32 dgst_len; + u32 W_len; + +} pdf17l8_tmp_t; + +typedef struct +{ + u32 digest_buf[4]; + +} phpass_tmp_t; + +typedef struct +{ + u32 digest_buf[4]; + +} md5crypt_tmp_t; + +typedef struct +{ + u32 alt_result[8]; + + u32 p_bytes[4]; + u32 s_bytes[4]; + +} sha256crypt_tmp_t; + +typedef struct +{ + u64 l_alt_result[8]; + + u64 l_p_bytes[2]; + u64 l_s_bytes[2]; + +} sha512crypt_tmp_t; + +typedef struct +{ + u32 ipad[5]; + u32 opad[5]; + + u32 dgst[10]; + u32 out[10]; + +} wpa_tmp_t; + +typedef struct +{ + u64 dgst[8]; + +} bitcoin_wallet_tmp_t; + +typedef struct +{ + u32 ipad[5]; + u32 opad[5]; + + u32 dgst[5]; + u32 out[4]; + +} dcc2_tmp_t; + +typedef struct +{ + u32 E[18]; + + u32 P[18]; + + u32 S0[256]; + u32 S1[256]; + u32 S2[256]; + u32 S3[256]; + +} bcrypt_tmp_t; + +typedef struct +{ + u32 digest[2]; + + u32 P[18]; + + u32 S0[256]; + u32 S1[256]; + u32 S2[256]; + u32 S3[256]; + +} pwsafe2_tmp_t; + +typedef struct +{ + u32 digest_buf[8]; + +} pwsafe3_tmp_t; + +typedef struct +{ + u32 digest_buf[5]; + +} androidpin_tmp_t; + +typedef struct +{ + u32 ipad[5]; + u32 opad[5]; + + u32 dgst[10]; + u32 out[10]; + +} androidfde_tmp_t; + +typedef struct +{ + u32 ipad[16]; + u32 opad[16]; + + u32 dgst[64]; + u32 out[64]; + +} tc_tmp_t; + +typedef struct +{ + u64 ipad[8]; + u64 opad[8]; + + u64 dgst[32]; + u64 out[32]; + +} tc64_tmp_t; + +typedef struct +{ + u32 ipad[4]; + u32 opad[4]; + + u32 dgst[32]; + u32 out[32]; + +} pbkdf2_md5_tmp_t; + +typedef struct +{ + u32 ipad[5]; + u32 opad[5]; + + u32 dgst[32]; + u32 out[32]; + +} pbkdf2_sha1_tmp_t; + +typedef struct +{ + u32 ipad[8]; + u32 opad[8]; + + u32 dgst[32]; + u32 out[32]; + +} pbkdf2_sha256_tmp_t; + +typedef struct +{ + u64 ipad[8]; + u64 opad[8]; + + u64 dgst[16]; + u64 out[16]; + +} pbkdf2_sha512_tmp_t; + +typedef struct +{ + u64 out[8]; + +} ecryptfs_tmp_t; + +typedef struct +{ + u64 ipad[8]; + u64 opad[8]; + + u64 dgst[16]; + u64 out[16]; + +} oraclet_tmp_t; + +typedef struct +{ + u32 ipad[5]; + u32 opad[5]; + + u32 dgst[5]; + u32 out[5]; + +} agilekey_tmp_t; + +typedef struct +{ + u32 ipad[5]; + u32 opad[5]; + + u32 dgst1[5]; + u32 out1[5]; + + u32 dgst2[5]; + u32 out2[5]; + +} mywallet_tmp_t; + +typedef struct +{ + u32 ipad[5]; + u32 opad[5]; + + u32 dgst[5]; + u32 out[5]; + +} sha1aix_tmp_t; + +typedef struct +{ + u32 ipad[8]; + u32 opad[8]; + + u32 dgst[8]; + u32 out[8]; + +} sha256aix_tmp_t; + +typedef struct +{ + u64 ipad[8]; + u64 opad[8]; + + u64 dgst[8]; + u64 out[8]; + +} sha512aix_tmp_t; + +typedef struct +{ + u32 ipad[8]; + u32 opad[8]; + + u32 dgst[8]; + u32 out[8]; + +} lastpass_tmp_t; + +typedef struct +{ + u64 digest_buf[8]; + +} drupal7_tmp_t; + +typedef struct +{ + u32 ipad[5]; + u32 opad[5]; + + u32 dgst[5]; + u32 out[5]; + +} lotus8_tmp_t; + +typedef struct +{ + u32 out[5]; + +} office2007_tmp_t; + +typedef struct +{ + u32 out[5]; + +} office2010_tmp_t; + +typedef struct +{ + u64 out[8]; + +} office2013_tmp_t; + +typedef struct +{ + u32 digest_buf[5]; + +} saph_sha1_tmp_t; + +typedef struct +{ + u32 block[16]; + + u32 dgst[8]; + + u32 block_len; + u32 final_len; + +} seven_zip_tmp_t; + +typedef struct +{ + u32 KEK[5]; + + u32 lsb[4]; + u32 cipher[4]; + +} axcrypt_tmp_t; + +typedef struct +{ + u32 tmp_digest[8]; + +} keepass_tmp_t; + +typedef struct +{ + u32 Kc[16]; + u32 Kd[16]; + + u32 iv[2]; + +} bsdicrypt_tmp_t; + +typedef struct +{ + u32 dgst[17][5]; + +} rar3_tmp_t; + +typedef struct +{ + u32 user[16]; + +} cram_md5_t; + +typedef struct +{ + u32 iv_buf[4]; + u32 iv_len; + + u32 salt_buf[4]; + u32 salt_len; + + u32 crc; + + u32 data_buf[96]; + u32 data_len; + + u32 unpack_size; + +} seven_zip_t; + +typedef struct +{ + u32 key; + u64 val; + +} hcstat_table_t; + +typedef struct +{ + u32 cs_buf[0x100]; + u32 cs_len; + +} cs_t; + +typedef struct +{ + u32 cmds[0x100]; + +} kernel_rule_t; + +typedef struct +{ + u32 salt_pos; + u32 digest_pos; + u32 hash_pos; + u32 gidvid; + u32 il_pos; + +} plain_t; + +typedef struct +{ + u32 i[16]; + + u32 pw_len; + + u32 alignment_placeholder_1; + u32 alignment_placeholder_2; + u32 alignment_placeholder_3; + +} pw_t; + +typedef struct +{ + u32 i; + +} bf_t; + +typedef struct +{ + u32 i[8]; + + u32 pw_len; + +} comb_t; + +typedef struct +{ + u32 b[32]; + +} bs_word_t; + +typedef struct +{ + uint4 P[64]; + +} scrypt_tmp_t; diff --git a/OpenCL/inc_vendor.cl b/OpenCL/inc_vendor.cl new file mode 100644 index 0000000..3adde96 --- /dev/null +++ b/OpenCL/inc_vendor.cl @@ -0,0 +1,237 @@ +/** + * Author......: Jens Steube + * License.....: MIT + */ + +//fails on intel opencl sdk +//#pragma OPENCL EXTENSION cl_khr_int64_base_atomics : enable +//#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable + +/** + * device type + */ + +#define DEVICE_TYPE_CPU 2 +#define DEVICE_TYPE_GPU 4 +#define DEVICE_TYPE_ACCEL 8 + +#if DEVICE_TYPE == DEVICE_TYPE_CPU +#define IS_CPU +#elif DEVICE_TYPE == DEVICE_TYPE_GPU +#define IS_GPU +#elif DEVICE_TYPE == DEVICE_TYPE_ACCEL +#define IS_ACCEL +#endif + +/** + * vendor specific + */ + +#if VENDOR_ID == (1 << 0) +#define IS_AMD +//#define IS_GENERIC +#elif VENDOR_ID == (1 << 1) +#define IS_APPLE +#define IS_GENERIC +#elif VENDOR_ID == (1 << 2) +#define IS_INTEL_BEIGNET +#define IS_GENERIC +#elif VENDOR_ID == (1 << 3) +#define IS_INTEL_SDK +#define IS_GENERIC +#elif VENDOR_ID == (1 << 4) +#define IS_MESA +#define IS_GENERIC +#elif VENDOR_ID == (1 << 5) +#define IS_NV +//#define IS_GENERIC +#elif VENDOR_ID == (1 << 6) +#define IS_POCL +#define IS_GENERIC +#else +#define IS_GENERIC +#endif + +/** + * AMD specific + */ + +#ifdef IS_AMD +#pragma OPENCL EXTENSION cl_amd_media_ops : enable +#pragma OPENCL EXTENSION cl_amd_media_ops2 : enable +#endif + +/** + * Unrolling is generally enabled, for all device types and hash modes + * There's a few exception when it's better not to unroll + */ + +// Some algorithms run into too much register pressure due to loop unrolling + +#ifdef IS_NV +#ifdef IS_GPU + +#if KERN_TYPE == 1500 +#undef _unroll +#endif +#if KERN_TYPE == 1800 +#undef _unroll +#endif +#if KERN_TYPE == 3000 +#undef _unroll +#endif +#if KERN_TYPE == 6221 +#undef _unroll +#endif +#if KERN_TYPE == 6222 +#undef _unroll +#endif +#if KERN_TYPE == 6223 +#undef _unroll +#endif +#if KERN_TYPE == 6500 +#undef _unroll +#endif +#if KERN_TYPE == 7100 +#undef _unroll +#endif +#if KERN_TYPE == 7400 +#undef _unroll +#endif +#if KERN_TYPE == 7900 +#undef _unroll +#endif +#if KERN_TYPE == 8200 +#undef _unroll +#endif +#if KERN_TYPE == 10400 +#undef _unroll +#endif +#if KERN_TYPE == 10500 +#undef _unroll +#endif +#if KERN_TYPE == 10700 +#undef _unroll +#endif +#if KERN_TYPE == 12300 +#undef _unroll +#endif +#if KERN_TYPE == 12400 +#undef _unroll +#endif +#if KERN_TYPE == 13721 +#undef _unroll +#endif +#if KERN_TYPE == 13722 +#undef _unroll +#endif +#if KERN_TYPE == 13723 +#undef _unroll +#endif + +#endif +#endif + +#ifdef IS_AMD +#ifdef IS_GPU + +#if KERN_TYPE == 3200 +#undef _unroll +#endif +#if KERN_TYPE == 5200 +#undef _unroll +#endif +#if KERN_TYPE == 6100 +#undef _unroll +#endif +#if KERN_TYPE == 6221 +#undef _unroll +#endif +#if KERN_TYPE == 6222 +#undef _unroll +#endif +#if KERN_TYPE == 6223 +#undef _unroll +#endif +#if KERN_TYPE == 6400 +#undef _unroll +#endif +#if KERN_TYPE == 6500 +#undef _unroll +#endif +#if KERN_TYPE == 6800 +#undef _unroll +#endif +#if KERN_TYPE == 7100 +#undef _unroll +#endif +#if KERN_TYPE == 7400 +#undef _unroll +#endif +#if KERN_TYPE == 8000 +#undef _unroll +#endif +#if KERN_TYPE == 8200 +#undef _unroll +#endif +#if KERN_TYPE == 10900 +#undef _unroll +#endif +#if KERN_TYPE == 11600 +#undef _unroll +#endif +#if KERN_TYPE == 12300 +#undef _unroll +#endif +#if KERN_TYPE == 12800 +#undef _unroll +#endif +#if KERN_TYPE == 12900 +#undef _unroll +#endif +#if KERN_TYPE == 13000 +#undef _unroll +#endif +#if KERN_TYPE == 13721 +#undef _unroll +#endif +#if KERN_TYPE == 13722 +#undef _unroll +#endif +#if KERN_TYPE == 13723 +#undef _unroll +#endif + +#endif +#endif + +// Some algorithms break due to loop unrolling, it's unknown why, probably compiler bugs +// Can overlap with above cases + +#ifdef IS_AMD +#ifdef IS_GPU + +#if KERN_TYPE == 1750 +#undef _unroll +#endif +#if KERN_TYPE == 1760 +#undef _unroll +#endif +#if KERN_TYPE == 6500 +#undef _unroll +#endif +#if KERN_TYPE == 7100 +#undef _unroll +#endif +#if KERN_TYPE == 9600 +#undef _unroll +#endif +#if KERN_TYPE == 12200 +#undef _unroll +#endif +#if KERN_TYPE == 12300 +#undef _unroll +#endif + +#endif +#endif diff --git a/OpenCL/kernel_aes256.c b/OpenCL/kernel_aes256.c deleted file mode 100644 index b3b6107..0000000 --- a/OpenCL/kernel_aes256.c +++ /dev/null @@ -1,1048 +0,0 @@ -/** - * Author......: Jens Steube - * License.....: MIT - */ - -__constant u32 te0[256] = -{ - 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, - 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, - 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, - 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, - 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, - 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, - 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, - 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, - 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, - 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, - 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, - 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, - 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, - 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, - 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, - 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, - 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, - 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, - 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, - 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, - 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, - 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, - 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, - 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, - 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, - 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, - 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, - 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, - 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, - 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, - 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, - 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d, - 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, - 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, - 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, - 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, - 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, - 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, - 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, - 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, - 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, - 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, - 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, - 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b, - 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, - 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, - 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, - 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, - 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, - 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, - 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, - 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, - 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, - 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, - 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, - 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, - 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, - 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, - 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, - 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, - 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, - 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, - 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, - 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, -}; - -__constant u32 te1[256] = -{ - 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, - 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, - 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, - 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, - 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, - 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, - 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, - 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, - 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, - 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, - 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, - 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, - 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, - 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, - 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, - 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, - 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, - 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, - 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, - 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, - 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, - 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, - 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, - 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, - 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, - 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, - 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, - 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, - 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, - 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, - 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, - 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, - 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, - 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, - 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, - 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, - 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, - 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, - 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, - 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, - 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, - 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, - 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, - 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, - 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, - 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, - 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, - 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, - 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, - 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, - 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, - 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, - 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, - 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, - 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, - 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, - 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, - 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, - 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, - 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, - 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, - 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, - 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, - 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, -}; - -__constant u32 te2[256] = -{ - 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, - 0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5, - 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, - 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, - 0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d, - 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, - 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, - 0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0, - 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, - 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, - 0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1, - 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, - 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, - 0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a, - 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, - 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, - 0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a, - 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, - 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, - 0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384, - 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, - 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, - 0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239, - 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, - 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, - 0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185, - 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, - 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, - 0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f, - 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, - 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, - 0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2, - 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, - 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, - 0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d, - 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, - 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, - 0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88, - 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, - 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, - 0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a, - 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, - 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, - 0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279, - 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, - 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, - 0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea, - 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, - 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, - 0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6, - 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, - 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, - 0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66, - 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, - 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, - 0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e, - 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, - 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, - 0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9, - 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, - 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, - 0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068, - 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, - 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, -}; - -__constant u32 te3[256] = -{ - 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, - 0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491, - 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, - 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, - 0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa, - 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, - 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, - 0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b, - 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, - 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, - 0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9, - 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, - 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, - 0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f, - 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, - 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, - 0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34, - 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, - 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, - 0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713, - 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, - 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, - 0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72, - 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, - 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, - 0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411, - 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, - 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, - 0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05, - 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, - 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, - 0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf, - 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, - 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, - 0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a, - 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, - 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, - 0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b, - 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, - 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, - 0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14, - 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, - 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, - 0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2, - 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, - 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, - 0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf, - 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, - 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, - 0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197, - 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, - 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, - 0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc, - 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, - 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, - 0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927, - 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, - 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, - 0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9, - 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, - 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, - 0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0, - 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, - 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, -}; - -__constant u32 te4[256] = -{ - 0x63636363, 0x7c7c7c7c, 0x77777777, 0x7b7b7b7b, - 0xf2f2f2f2, 0x6b6b6b6b, 0x6f6f6f6f, 0xc5c5c5c5, - 0x30303030, 0x01010101, 0x67676767, 0x2b2b2b2b, - 0xfefefefe, 0xd7d7d7d7, 0xabababab, 0x76767676, - 0xcacacaca, 0x82828282, 0xc9c9c9c9, 0x7d7d7d7d, - 0xfafafafa, 0x59595959, 0x47474747, 0xf0f0f0f0, - 0xadadadad, 0xd4d4d4d4, 0xa2a2a2a2, 0xafafafaf, - 0x9c9c9c9c, 0xa4a4a4a4, 0x72727272, 0xc0c0c0c0, - 0xb7b7b7b7, 0xfdfdfdfd, 0x93939393, 0x26262626, - 0x36363636, 0x3f3f3f3f, 0xf7f7f7f7, 0xcccccccc, - 0x34343434, 0xa5a5a5a5, 0xe5e5e5e5, 0xf1f1f1f1, - 0x71717171, 0xd8d8d8d8, 0x31313131, 0x15151515, - 0x04040404, 0xc7c7c7c7, 0x23232323, 0xc3c3c3c3, - 0x18181818, 0x96969696, 0x05050505, 0x9a9a9a9a, - 0x07070707, 0x12121212, 0x80808080, 0xe2e2e2e2, - 0xebebebeb, 0x27272727, 0xb2b2b2b2, 0x75757575, - 0x09090909, 0x83838383, 0x2c2c2c2c, 0x1a1a1a1a, - 0x1b1b1b1b, 0x6e6e6e6e, 0x5a5a5a5a, 0xa0a0a0a0, - 0x52525252, 0x3b3b3b3b, 0xd6d6d6d6, 0xb3b3b3b3, - 0x29292929, 0xe3e3e3e3, 0x2f2f2f2f, 0x84848484, - 0x53535353, 0xd1d1d1d1, 0x00000000, 0xedededed, - 0x20202020, 0xfcfcfcfc, 0xb1b1b1b1, 0x5b5b5b5b, - 0x6a6a6a6a, 0xcbcbcbcb, 0xbebebebe, 0x39393939, - 0x4a4a4a4a, 0x4c4c4c4c, 0x58585858, 0xcfcfcfcf, - 0xd0d0d0d0, 0xefefefef, 0xaaaaaaaa, 0xfbfbfbfb, - 0x43434343, 0x4d4d4d4d, 0x33333333, 0x85858585, - 0x45454545, 0xf9f9f9f9, 0x02020202, 0x7f7f7f7f, - 0x50505050, 0x3c3c3c3c, 0x9f9f9f9f, 0xa8a8a8a8, - 0x51515151, 0xa3a3a3a3, 0x40404040, 0x8f8f8f8f, - 0x92929292, 0x9d9d9d9d, 0x38383838, 0xf5f5f5f5, - 0xbcbcbcbc, 0xb6b6b6b6, 0xdadadada, 0x21212121, - 0x10101010, 0xffffffff, 0xf3f3f3f3, 0xd2d2d2d2, - 0xcdcdcdcd, 0x0c0c0c0c, 0x13131313, 0xecececec, - 0x5f5f5f5f, 0x97979797, 0x44444444, 0x17171717, - 0xc4c4c4c4, 0xa7a7a7a7, 0x7e7e7e7e, 0x3d3d3d3d, - 0x64646464, 0x5d5d5d5d, 0x19191919, 0x73737373, - 0x60606060, 0x81818181, 0x4f4f4f4f, 0xdcdcdcdc, - 0x22222222, 0x2a2a2a2a, 0x90909090, 0x88888888, - 0x46464646, 0xeeeeeeee, 0xb8b8b8b8, 0x14141414, - 0xdededede, 0x5e5e5e5e, 0x0b0b0b0b, 0xdbdbdbdb, - 0xe0e0e0e0, 0x32323232, 0x3a3a3a3a, 0x0a0a0a0a, - 0x49494949, 0x06060606, 0x24242424, 0x5c5c5c5c, - 0xc2c2c2c2, 0xd3d3d3d3, 0xacacacac, 0x62626262, - 0x91919191, 0x95959595, 0xe4e4e4e4, 0x79797979, - 0xe7e7e7e7, 0xc8c8c8c8, 0x37373737, 0x6d6d6d6d, - 0x8d8d8d8d, 0xd5d5d5d5, 0x4e4e4e4e, 0xa9a9a9a9, - 0x6c6c6c6c, 0x56565656, 0xf4f4f4f4, 0xeaeaeaea, - 0x65656565, 0x7a7a7a7a, 0xaeaeaeae, 0x08080808, - 0xbabababa, 0x78787878, 0x25252525, 0x2e2e2e2e, - 0x1c1c1c1c, 0xa6a6a6a6, 0xb4b4b4b4, 0xc6c6c6c6, - 0xe8e8e8e8, 0xdddddddd, 0x74747474, 0x1f1f1f1f, - 0x4b4b4b4b, 0xbdbdbdbd, 0x8b8b8b8b, 0x8a8a8a8a, - 0x70707070, 0x3e3e3e3e, 0xb5b5b5b5, 0x66666666, - 0x48484848, 0x03030303, 0xf6f6f6f6, 0x0e0e0e0e, - 0x61616161, 0x35353535, 0x57575757, 0xb9b9b9b9, - 0x86868686, 0xc1c1c1c1, 0x1d1d1d1d, 0x9e9e9e9e, - 0xe1e1e1e1, 0xf8f8f8f8, 0x98989898, 0x11111111, - 0x69696969, 0xd9d9d9d9, 0x8e8e8e8e, 0x94949494, - 0x9b9b9b9b, 0x1e1e1e1e, 0x87878787, 0xe9e9e9e9, - 0xcececece, 0x55555555, 0x28282828, 0xdfdfdfdf, - 0x8c8c8c8c, 0xa1a1a1a1, 0x89898989, 0x0d0d0d0d, - 0xbfbfbfbf, 0xe6e6e6e6, 0x42424242, 0x68686868, - 0x41414141, 0x99999999, 0x2d2d2d2d, 0x0f0f0f0f, - 0xb0b0b0b0, 0x54545454, 0xbbbbbbbb, 0x16161616, -}; - -__constant u32 td0[256] = -{ - 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, - 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, - 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, - 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, - 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, - 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, - 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, - 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, - 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, - 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, - 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, - 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94, - 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, - 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, - 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, - 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c, - 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, - 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, - 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, - 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, - 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, - 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, - 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, - 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, - 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, - 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, - 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, - 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a, - 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, - 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, - 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, - 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8, - 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, - 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, - 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, - 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, - 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, - 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, - 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, - 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, - 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, - 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, - 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, - 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5, - 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, - 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, - 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, - 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6, - 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, - 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, - 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, - 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, - 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, - 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, - 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, - 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, - 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, - 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, - 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, - 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86, - 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, - 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, - 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, - 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742, -}; - -__constant u32 td1[256] = -{ - 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, - 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, - 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, - 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, - 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, - 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, - 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, - 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, - 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, - 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, - 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, - 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, - 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, - 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, - 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, - 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, - 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, - 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, - 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, - 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, - 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, - 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, - 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, - 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, - 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, - 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, - 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, - 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, - 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, - 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, - 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, - 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, - 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, - 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, - 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, - 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, - 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, - 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, - 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, - 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, - 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, - 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, - 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, - 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, - 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, - 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, - 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, - 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, - 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, - 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, - 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, - 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, - 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, - 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, - 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, - 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, - 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, - 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, - 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, - 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, - 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, - 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, - 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, - 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, -}; - -__constant u32 td2[256] = -{ - 0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27, - 0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3, - 0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502, - 0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562, - 0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe, - 0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3, - 0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552, - 0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9, - 0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9, - 0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce, - 0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253, - 0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908, - 0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b, - 0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655, - 0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337, - 0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16, - 0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69, - 0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6, - 0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6, - 0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e, - 0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6, - 0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050, - 0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9, - 0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8, - 0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000, - 0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a, - 0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d, - 0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436, - 0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b, - 0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12, - 0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b, - 0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e, - 0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f, - 0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb, - 0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4, - 0xdccad731, 0x85104263, 0x22401397, 0x112084c6, - 0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729, - 0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1, - 0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9, - 0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233, - 0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4, - 0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad, - 0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e, - 0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3, - 0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25, - 0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b, - 0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f, - 0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15, - 0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0, - 0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2, - 0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7, - 0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791, - 0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496, - 0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665, - 0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b, - 0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6, - 0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13, - 0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47, - 0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7, - 0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844, - 0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3, - 0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d, - 0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456, - 0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8, -}; - -__constant u32 td3[256] = -{ - 0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a, - 0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b, - 0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5, - 0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5, - 0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d, - 0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b, - 0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95, - 0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e, - 0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27, - 0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d, - 0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562, - 0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9, - 0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752, - 0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66, - 0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3, - 0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced, - 0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e, - 0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4, - 0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4, - 0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd, - 0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d, - 0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60, - 0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767, - 0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79, - 0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000, - 0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c, - 0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736, - 0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24, - 0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b, - 0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c, - 0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12, - 0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814, - 0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3, - 0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b, - 0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8, - 0x31dccad7, 0x63851042, 0x97224013, 0xc6112084, - 0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7, - 0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077, - 0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247, - 0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22, - 0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698, - 0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f, - 0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254, - 0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582, - 0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf, - 0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb, - 0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883, - 0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef, - 0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629, - 0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035, - 0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533, - 0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17, - 0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4, - 0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46, - 0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb, - 0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d, - 0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb, - 0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a, - 0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73, - 0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678, - 0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2, - 0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff, - 0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064, - 0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0, -}; - -__constant u32 td4[256] = -{ - 0x52525252, 0x09090909, 0x6a6a6a6a, 0xd5d5d5d5, - 0x30303030, 0x36363636, 0xa5a5a5a5, 0x38383838, - 0xbfbfbfbf, 0x40404040, 0xa3a3a3a3, 0x9e9e9e9e, - 0x81818181, 0xf3f3f3f3, 0xd7d7d7d7, 0xfbfbfbfb, - 0x7c7c7c7c, 0xe3e3e3e3, 0x39393939, 0x82828282, - 0x9b9b9b9b, 0x2f2f2f2f, 0xffffffff, 0x87878787, - 0x34343434, 0x8e8e8e8e, 0x43434343, 0x44444444, - 0xc4c4c4c4, 0xdededede, 0xe9e9e9e9, 0xcbcbcbcb, - 0x54545454, 0x7b7b7b7b, 0x94949494, 0x32323232, - 0xa6a6a6a6, 0xc2c2c2c2, 0x23232323, 0x3d3d3d3d, - 0xeeeeeeee, 0x4c4c4c4c, 0x95959595, 0x0b0b0b0b, - 0x42424242, 0xfafafafa, 0xc3c3c3c3, 0x4e4e4e4e, - 0x08080808, 0x2e2e2e2e, 0xa1a1a1a1, 0x66666666, - 0x28282828, 0xd9d9d9d9, 0x24242424, 0xb2b2b2b2, - 0x76767676, 0x5b5b5b5b, 0xa2a2a2a2, 0x49494949, - 0x6d6d6d6d, 0x8b8b8b8b, 0xd1d1d1d1, 0x25252525, - 0x72727272, 0xf8f8f8f8, 0xf6f6f6f6, 0x64646464, - 0x86868686, 0x68686868, 0x98989898, 0x16161616, - 0xd4d4d4d4, 0xa4a4a4a4, 0x5c5c5c5c, 0xcccccccc, - 0x5d5d5d5d, 0x65656565, 0xb6b6b6b6, 0x92929292, - 0x6c6c6c6c, 0x70707070, 0x48484848, 0x50505050, - 0xfdfdfdfd, 0xedededed, 0xb9b9b9b9, 0xdadadada, - 0x5e5e5e5e, 0x15151515, 0x46464646, 0x57575757, - 0xa7a7a7a7, 0x8d8d8d8d, 0x9d9d9d9d, 0x84848484, - 0x90909090, 0xd8d8d8d8, 0xabababab, 0x00000000, - 0x8c8c8c8c, 0xbcbcbcbc, 0xd3d3d3d3, 0x0a0a0a0a, - 0xf7f7f7f7, 0xe4e4e4e4, 0x58585858, 0x05050505, - 0xb8b8b8b8, 0xb3b3b3b3, 0x45454545, 0x06060606, - 0xd0d0d0d0, 0x2c2c2c2c, 0x1e1e1e1e, 0x8f8f8f8f, - 0xcacacaca, 0x3f3f3f3f, 0x0f0f0f0f, 0x02020202, - 0xc1c1c1c1, 0xafafafaf, 0xbdbdbdbd, 0x03030303, - 0x01010101, 0x13131313, 0x8a8a8a8a, 0x6b6b6b6b, - 0x3a3a3a3a, 0x91919191, 0x11111111, 0x41414141, - 0x4f4f4f4f, 0x67676767, 0xdcdcdcdc, 0xeaeaeaea, - 0x97979797, 0xf2f2f2f2, 0xcfcfcfcf, 0xcececece, - 0xf0f0f0f0, 0xb4b4b4b4, 0xe6e6e6e6, 0x73737373, - 0x96969696, 0xacacacac, 0x74747474, 0x22222222, - 0xe7e7e7e7, 0xadadadad, 0x35353535, 0x85858585, - 0xe2e2e2e2, 0xf9f9f9f9, 0x37373737, 0xe8e8e8e8, - 0x1c1c1c1c, 0x75757575, 0xdfdfdfdf, 0x6e6e6e6e, - 0x47474747, 0xf1f1f1f1, 0x1a1a1a1a, 0x71717171, - 0x1d1d1d1d, 0x29292929, 0xc5c5c5c5, 0x89898989, - 0x6f6f6f6f, 0xb7b7b7b7, 0x62626262, 0x0e0e0e0e, - 0xaaaaaaaa, 0x18181818, 0xbebebebe, 0x1b1b1b1b, - 0xfcfcfcfc, 0x56565656, 0x3e3e3e3e, 0x4b4b4b4b, - 0xc6c6c6c6, 0xd2d2d2d2, 0x79797979, 0x20202020, - 0x9a9a9a9a, 0xdbdbdbdb, 0xc0c0c0c0, 0xfefefefe, - 0x78787878, 0xcdcdcdcd, 0x5a5a5a5a, 0xf4f4f4f4, - 0x1f1f1f1f, 0xdddddddd, 0xa8a8a8a8, 0x33333333, - 0x88888888, 0x07070707, 0xc7c7c7c7, 0x31313131, - 0xb1b1b1b1, 0x12121212, 0x10101010, 0x59595959, - 0x27272727, 0x80808080, 0xecececec, 0x5f5f5f5f, - 0x60606060, 0x51515151, 0x7f7f7f7f, 0xa9a9a9a9, - 0x19191919, 0xb5b5b5b5, 0x4a4a4a4a, 0x0d0d0d0d, - 0x2d2d2d2d, 0xe5e5e5e5, 0x7a7a7a7a, 0x9f9f9f9f, - 0x93939393, 0xc9c9c9c9, 0x9c9c9c9c, 0xefefefef, - 0xa0a0a0a0, 0xe0e0e0e0, 0x3b3b3b3b, 0x4d4d4d4d, - 0xaeaeaeae, 0x2a2a2a2a, 0xf5f5f5f5, 0xb0b0b0b0, - 0xc8c8c8c8, 0xebebebeb, 0xbbbbbbbb, 0x3c3c3c3c, - 0x83838383, 0x53535353, 0x99999999, 0x61616161, - 0x17171717, 0x2b2b2b2b, 0x04040404, 0x7e7e7e7e, - 0xbabababa, 0x77777777, 0xd6d6d6d6, 0x26262626, - 0xe1e1e1e1, 0x69696969, 0x14141414, 0x63636363, - 0x55555555, 0x21212121, 0x0c0c0c0c, 0x7d7d7d7d, -}; - -__constant u32 rcon[] = -{ - 0x01000000, 0x02000000, 0x04000000, 0x08000000, - 0x10000000, 0x20000000, 0x40000000, 0x80000000, - 0x1b000000, 0x36000000, -}; - -void aes256_ExpandKey (u32 *ks, const u32 *ukey) -{ - ks[0] = ukey[0]; - ks[1] = ukey[1]; - ks[2] = ukey[2]; - ks[3] = ukey[3]; - ks[4] = ukey[4]; - ks[5] = ukey[5]; - ks[6] = ukey[6]; - ks[7] = ukey[7]; - - int i; - int j; - - i = 0; - j = 0; - - while (1) - { - u32 temp = ks[j + 7]; - - ks[j + 8] = ks[j + 0] - ^ (te2[(temp >> 16) & 0xff] & 0xff000000) - ^ (te3[(temp >> 8) & 0xff] & 0x00ff0000) - ^ (te0[(temp >> 0) & 0xff] & 0x0000ff00) - ^ (te1[(temp >> 24) & 0xff] & 0x000000ff) - ^ rcon[i]; - - ks[j + 9] = ks[j + 1] ^ ks[j + 8]; - ks[j + 10] = ks[j + 2] ^ ks[j + 9]; - ks[j + 11] = ks[j + 3] ^ ks[j + 10]; - - if (++i == 7) break; - - temp = ks[j + 11]; - - ks[j + 12] = ks[j + 4] - ^ (te2[(temp >> 24) & 0xff] & 0xff000000) - ^ (te3[(temp >> 16) & 0xff] & 0x00ff0000) - ^ (te0[(temp >> 8) & 0xff] & 0x0000ff00) - ^ (te1[(temp >> 0) & 0xff] & 0x000000ff); - - ks[j + 13] = ks[j + 5] ^ ks[j + 12]; - ks[j + 14] = ks[j + 6] ^ ks[j + 13]; - ks[j + 15] = ks[j + 7] ^ ks[j + 14]; - - j += 8; - } -} - -void aes256_InvertKey (u32 *ks) -{ - for (u32 i = 0, j = 56; i < j; i += 4, j -= 4) - { - u32 temp; - - temp = ks[i + 0]; ks[i + 0] = ks[j + 0]; ks[j + 0] = temp; - temp = ks[i + 1]; ks[i + 1] = ks[j + 1]; ks[j + 1] = temp; - temp = ks[i + 2]; ks[i + 2] = ks[j + 2]; ks[j + 2] = temp; - temp = ks[i + 3]; ks[i + 3] = ks[j + 3]; ks[j + 3] = temp; - } - - for (u32 i = 1, j = 4; i < 14; i += 1, j += 4) - { - ks[j + 0] = - td0[te1[(ks[j + 0] >> 24) & 0xff] & 0xff] ^ - td1[te1[(ks[j + 0] >> 16) & 0xff] & 0xff] ^ - td2[te1[(ks[j + 0] >> 8) & 0xff] & 0xff] ^ - td3[te1[(ks[j + 0] >> 0) & 0xff] & 0xff]; - - ks[j + 1] = - td0[te1[(ks[j + 1] >> 24) & 0xff] & 0xff] ^ - td1[te1[(ks[j + 1] >> 16) & 0xff] & 0xff] ^ - td2[te1[(ks[j + 1] >> 8) & 0xff] & 0xff] ^ - td3[te1[(ks[j + 1] >> 0) & 0xff] & 0xff]; - - ks[j + 2] = - td0[te1[(ks[j + 2] >> 24) & 0xff] & 0xff] ^ - td1[te1[(ks[j + 2] >> 16) & 0xff] & 0xff] ^ - td2[te1[(ks[j + 2] >> 8) & 0xff] & 0xff] ^ - td3[te1[(ks[j + 2] >> 0) & 0xff] & 0xff]; - - ks[j + 3] = - td0[te1[(ks[j + 3] >> 24) & 0xff] & 0xff] ^ - td1[te1[(ks[j + 3] >> 16) & 0xff] & 0xff] ^ - td2[te1[(ks[j + 3] >> 8) & 0xff] & 0xff] ^ - td3[te1[(ks[j + 3] >> 0) & 0xff] & 0xff]; - } -} - -void aes256_set_encrypt_key (u32 *ks, const u32 *ukey) -{ - u32 ukey_s[8]; - - ukey_s[0] = swap32 (ukey[0]); - ukey_s[1] = swap32 (ukey[1]); - ukey_s[2] = swap32 (ukey[2]); - ukey_s[3] = swap32 (ukey[3]); - ukey_s[4] = swap32 (ukey[4]); - ukey_s[5] = swap32 (ukey[5]); - ukey_s[6] = swap32 (ukey[6]); - ukey_s[7] = swap32 (ukey[7]); - - aes256_ExpandKey (ks, ukey_s); -} - -void aes256_set_decrypt_key (u32 *ks, const u32 *ukey) -{ - u32 ukey_s[8]; - - ukey_s[0] = swap32 (ukey[0]); - ukey_s[1] = swap32 (ukey[1]); - ukey_s[2] = swap32 (ukey[2]); - ukey_s[3] = swap32 (ukey[3]); - ukey_s[4] = swap32 (ukey[4]); - ukey_s[5] = swap32 (ukey[5]); - ukey_s[6] = swap32 (ukey[6]); - ukey_s[7] = swap32 (ukey[7]); - - aes256_ExpandKey (ks, ukey_s); - - aes256_InvertKey (ks); -} - -void aes256_decrypt (const u32 *ks, const u32 *in, u32 *out) -{ - u32 in_s[4]; - - in_s[0] = swap32 (in[0]); - in_s[1] = swap32 (in[1]); - in_s[2] = swap32 (in[2]); - in_s[3] = swap32 (in[3]); - - u32 s0 = in_s[0] ^ ks[0]; - u32 s1 = in_s[1] ^ ks[1]; - u32 s2 = in_s[2] ^ ks[2]; - u32 s3 = in_s[3] ^ ks[3]; - - u32 t0; - u32 t1; - u32 t2; - u32 t3; - - t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[ 4]; - t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[ 5]; - t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[ 6]; - t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[ 7]; - s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[ 8]; - s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[ 9]; - s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[10]; - s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[11]; - t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[12]; - t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[13]; - t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[14]; - t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[15]; - s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[16]; - s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[17]; - s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[18]; - s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[19]; - t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[20]; - t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[21]; - t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[22]; - t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[23]; - s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[24]; - s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[25]; - s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[26]; - s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[27]; - t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[28]; - t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[29]; - t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[30]; - t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[31]; - s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[32]; - s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[33]; - s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[34]; - s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[35]; - t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[36]; - t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[37]; - t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[38]; - t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[39]; - s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[40]; - s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[41]; - s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[42]; - s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[43]; - t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[44]; - t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[45]; - t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[46]; - t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[47]; - s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >> 8) & 0xff] ^ td3[t1 & 0xff] ^ ks[48]; - s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >> 8) & 0xff] ^ td3[t2 & 0xff] ^ ks[49]; - s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >> 8) & 0xff] ^ td3[t3 & 0xff] ^ ks[50]; - s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >> 8) & 0xff] ^ td3[t0 & 0xff] ^ ks[51]; - t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >> 8) & 0xff] ^ td3[s1 & 0xff] ^ ks[52]; - t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >> 8) & 0xff] ^ td3[s2 & 0xff] ^ ks[53]; - t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >> 8) & 0xff] ^ td3[s3 & 0xff] ^ ks[54]; - t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >> 8) & 0xff] ^ td3[s0 & 0xff] ^ ks[55]; - - out[0] = (td4[(t0 >> 24) & 0xff] & 0xff000000) - ^ (td4[(t3 >> 16) & 0xff] & 0x00ff0000) - ^ (td4[(t2 >> 8) & 0xff] & 0x0000ff00) - ^ (td4[(t1 >> 0) & 0xff] & 0x000000ff) - ^ ks[56]; - - out[1] = (td4[(t1 >> 24) & 0xff] & 0xff000000) - ^ (td4[(t0 >> 16) & 0xff] & 0x00ff0000) - ^ (td4[(t3 >> 8) & 0xff] & 0x0000ff00) - ^ (td4[(t2 >> 0) & 0xff] & 0x000000ff) - ^ ks[57]; - - out[2] = (td4[(t2 >> 24) & 0xff] & 0xff000000) - ^ (td4[(t1 >> 16) & 0xff] & 0x00ff0000) - ^ (td4[(t0 >> 8) & 0xff] & 0x0000ff00) - ^ (td4[(t3 >> 0) & 0xff] & 0x000000ff) - ^ ks[58]; - - out[3] = (td4[(t3 >> 24) & 0xff] & 0xff000000) - ^ (td4[(t2 >> 16) & 0xff] & 0x00ff0000) - ^ (td4[(t1 >> 8) & 0xff] & 0x0000ff00) - ^ (td4[(t0 >> 0) & 0xff] & 0x000000ff) - ^ ks[59]; - - out[0] = swap32 (out[0]); - out[1] = swap32 (out[1]); - out[2] = swap32 (out[2]); - out[3] = swap32 (out[3]); -} - -void aes256_encrypt (const u32 *ks, const u32 *in, u32 *out) -{ - u32 in_s[4]; - - in_s[0] = swap32 (in[0]); - in_s[1] = swap32 (in[1]); - in_s[2] = swap32 (in[2]); - in_s[3] = swap32 (in[3]); - - u32 s0 = in_s[0] ^ ks[0]; - u32 s1 = in_s[1] ^ ks[1]; - u32 s2 = in_s[2] ^ ks[2]; - u32 s3 = in_s[3] ^ ks[3]; - - u32 t0; - u32 t1; - u32 t2; - u32 t3; - - t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[ 4]; - t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[ 5]; - t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[ 6]; - t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[ 7]; - s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[ 8]; - s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[ 9]; - s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[10]; - s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[11]; - t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[12]; - t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[13]; - t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[14]; - t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[15]; - s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[16]; - s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[17]; - s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[18]; - s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[19]; - t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[20]; - t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[21]; - t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[22]; - t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[23]; - s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[24]; - s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[25]; - s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[26]; - s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[27]; - t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[28]; - t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[29]; - t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[30]; - t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[31]; - s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[32]; - s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[33]; - s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[34]; - s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[35]; - t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[36]; - t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[37]; - t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[38]; - t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[39]; - s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[40]; - s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[41]; - s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[42]; - s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[43]; - t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[44]; - t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[45]; - t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[46]; - t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[47]; - s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >> 8) & 0xff] ^ te3[t3 & 0xff] ^ ks[48]; - s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >> 8) & 0xff] ^ te3[t0 & 0xff] ^ ks[49]; - s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >> 8) & 0xff] ^ te3[t1 & 0xff] ^ ks[50]; - s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >> 8) & 0xff] ^ te3[t2 & 0xff] ^ ks[51]; - t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >> 8) & 0xff] ^ te3[s3 & 0xff] ^ ks[52]; - t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >> 8) & 0xff] ^ te3[s0 & 0xff] ^ ks[53]; - t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >> 8) & 0xff] ^ te3[s1 & 0xff] ^ ks[54]; - t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >> 8) & 0xff] ^ te3[s2 & 0xff] ^ ks[55]; - - out[0] = (te4[(t0 >> 24) & 0xff] & 0xff000000) - ^ (te4[(t1 >> 16) & 0xff] & 0x00ff0000) - ^ (te4[(t2 >> 8) & 0xff] & 0x0000ff00) - ^ (te4[(t3 >> 0) & 0xff] & 0x000000ff) - ^ ks[56]; - - out[1] = (te4[(t1 >> 24) & 0xff] & 0xff000000) - ^ (te4[(t2 >> 16) & 0xff] & 0x00ff0000) - ^ (te4[(t3 >> 8) & 0xff] & 0x0000ff00) - ^ (te4[(t0 >> 0) & 0xff] & 0x000000ff) - ^ ks[57]; - - out[2] = (te4[(t2 >> 24) & 0xff] & 0xff000000) - ^ (te4[(t3 >> 16) & 0xff] & 0x00ff0000) - ^ (te4[(t0 >> 8) & 0xff] & 0x0000ff00) - ^ (te4[(t1 >> 0) & 0xff] & 0x000000ff) - ^ ks[58]; - - out[3] = (te4[(t3 >> 24) & 0xff] & 0xff000000) - ^ (te4[(t0 >> 16) & 0xff] & 0x00ff0000) - ^ (te4[(t1 >> 8) & 0xff] & 0x0000ff00) - ^ (te4[(t2 >> 0) & 0xff] & 0x000000ff) - ^ ks[59]; - - out[0] = swap32 (out[0]); - out[1] = swap32 (out[1]); - out[2] = swap32 (out[2]); - out[3] = swap32 (out[3]); -} - -void aes256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out) -{ - u32 T[4] = { 0 }; - u32 Z[4] = { 0 }; - - out[0] = in[0]; - out[1] = in[1]; - out[2] = in[2]; - out[3] = in[3]; - - u32 ks[60]; - - aes256_set_encrypt_key (ks, ukey2); - aes256_encrypt (ks, Z, T); - - out[0] ^= T[0]; - out[1] ^= T[1]; - out[2] ^= T[2]; - out[3] ^= T[3]; - - aes256_set_decrypt_key (ks, ukey1); - aes256_decrypt (ks, out, out); - - out[0] ^= T[0]; - out[1] ^= T[1]; - out[2] ^= T[2]; - out[3] ^= T[3]; -} diff --git a/OpenCL/kernel_serpent256.c b/OpenCL/kernel_serpent256.c deleted file mode 100644 index 462c0b9..0000000 --- a/OpenCL/kernel_serpent256.c +++ /dev/null @@ -1,587 +0,0 @@ -/* This is an independent implementation of the encryption algorithm: */ -/* */ -/* Serpent by Ross Anderson, Eli Biham and Lars Knudsen */ -/* */ -/* which is a candidate algorithm in the Advanced Encryption Standard */ -/* programme of the US National Institute of Standards and Technology. */ -/* */ -/* Copyright in this implementation is held by Dr B R Gladman but I */ -/* hereby give permission for its free direct or derivative use subject */ -/* to acknowledgment of its origin and compliance with any conditions */ -/* that the originators of the algorithm place on its exploitation. */ -/* */ -/* Dr Brian Gladman (gladman@seven77.demon.co.uk) 14th January 1999 */ -/* */ -/* -------------------------------------------------------------------- */ -/* */ -/* Cleaned and optimized for GPU use with hashcat by Jens Steube */ - -/* 15 terms */ - -#define sb0(a,b,c,d,e,f,g,h) \ - t1 = a ^ d; \ - t2 = a & d; \ - t3 = c ^ t1; \ - t6 = b & t1; \ - t4 = b ^ t3; \ - t10 = ~t3; \ - h = t2 ^ t4; \ - t7 = a ^ t6; \ - t14 = ~t7; \ - t8 = c | t7; \ - t11 = t3 ^ t7; \ - g = t4 ^ t8; \ - t12 = h & t11; \ - f = t10 ^ t12; \ - e = t12 ^ t14 - -/* 15 terms */ - -#define ib0(a,b,c,d,e,f,g,h) \ - t1 = ~a; \ - t2 = a ^ b; \ - t3 = t1 | t2; \ - t4 = d ^ t3; \ - t7 = d & t2; \ - t5 = c ^ t4; \ - t8 = t1 ^ t7; \ - g = t2 ^ t5; \ - t11 = a & t4; \ - t9 = g & t8; \ - t14 = t5 ^ t8; \ - f = t4 ^ t9; \ - t12 = t5 | f; \ - h = t11 ^ t12; \ - e = h ^ t14 - -/* 14 terms! */ - -#define sb1(a,b,c,d,e,f,g,h) \ - t1 = ~a; \ - t2 = b ^ t1; \ - t3 = a | t2; \ - t4 = d | t2; \ - t5 = c ^ t3; \ - g = d ^ t5; \ - t7 = b ^ t4; \ - t8 = t2 ^ g; \ - t9 = t5 & t7; \ - h = t8 ^ t9; \ - t11 = t5 ^ t7; \ - f = h ^ t11; \ - t13 = t8 & t11; \ - e = t5 ^ t13 - -/* 17 terms */ - -#define ib1(a,b,c,d,e,f,g,h) \ - t1 = a ^ d; \ - t2 = a & b; \ - t3 = b ^ c; \ - t4 = a ^ t3; \ - t5 = b | d; \ - t7 = c | t1; \ - h = t4 ^ t5; \ - t8 = b ^ t7; \ - t11 = ~t2; \ - t9 = t4 & t8; \ - f = t1 ^ t9; \ - t13 = t9 ^ t11; \ - t12 = h & f; \ - g = t12 ^ t13; \ - t15 = a & d; \ - t16 = c ^ t13; \ - e = t15 ^ t16 - -/* 16 terms */ - -#define sb2(a,b,c,d,e,f,g,h) \ - t1 = ~a; \ - t2 = b ^ d; \ - t3 = c & t1; \ - t13 = d | t1; \ - e = t2 ^ t3; \ - t5 = c ^ t1; \ - t6 = c ^ e; \ - t7 = b & t6; \ - t10 = e | t5; \ - h = t5 ^ t7; \ - t9 = d | t7; \ - t11 = t9 & t10; \ - t14 = t2 ^ h; \ - g = a ^ t11; \ - t15 = g ^ t13; \ - f = t14 ^ t15 - -/* 16 terms */ - -#define ib2(a,b,c,d,e,f,g,h) \ - t1 = b ^ d; \ - t2 = ~t1; \ - t3 = a ^ c; \ - t4 = c ^ t1; \ - t7 = a | t2; \ - t5 = b & t4; \ - t8 = d ^ t7; \ - t11 = ~t4; \ - e = t3 ^ t5; \ - t9 = t3 | t8; \ - t14 = d & t11; \ - h = t1 ^ t9; \ - t12 = e | h; \ - f = t11 ^ t12; \ - t15 = t3 ^ t12; \ - g = t14 ^ t15 - -/* 17 terms */ - -#define sb3(a,b,c,d,e,f,g,h) \ - t1 = a ^ c; \ - t2 = d ^ t1; \ - t3 = a & t2; \ - t4 = d ^ t3; \ - t5 = b & t4; \ - g = t2 ^ t5; \ - t7 = a | g; \ - t8 = b | d; \ - t11 = a | d; \ - t9 = t4 & t7; \ - f = t8 ^ t9; \ - t12 = b ^ t11; \ - t13 = g ^ t9; \ - t15 = t3 ^ t8; \ - h = t12 ^ t13; \ - t16 = c & t15; \ - e = t12 ^ t16 - -/* 16 term solution that performs less well than 17 term one - in my environment (PPro/PII) - -#define sb3(a,b,c,d,e,f,g,h) \ - t1 = a ^ b; \ - t2 = a & c; \ - t3 = a | d; \ - t4 = c ^ d; \ - t5 = t1 & t3; \ - t6 = t2 | t5; \ - g = t4 ^ t6; \ - t8 = b ^ t3; \ - t9 = t6 ^ t8; \ - t10 = t4 & t9; \ - e = t1 ^ t10; \ - t12 = g & e; \ - f = t9 ^ t12; \ - t14 = b | d; \ - t15 = t4 ^ t12; \ - h = t14 ^ t15 -*/ - -/* 17 terms */ - -#define ib3(a,b,c,d,e,f,g,h) \ - t1 = b ^ c; \ - t2 = b | c; \ - t3 = a ^ c; \ - t7 = a ^ d; \ - t4 = t2 ^ t3; \ - t5 = d | t4; \ - t9 = t2 ^ t7; \ - e = t1 ^ t5; \ - t8 = t1 | t5; \ - t11 = a & t4; \ - g = t8 ^ t9; \ - t12 = e | t9; \ - f = t11 ^ t12; \ - t14 = a & g; \ - t15 = t2 ^ t14; \ - t16 = e & t15; \ - h = t4 ^ t16 - -/* 15 terms */ - -#define sb4(a,b,c,d,e,f,g,h) \ - t1 = a ^ d; \ - t2 = d & t1; \ - t3 = c ^ t2; \ - t4 = b | t3; \ - h = t1 ^ t4; \ - t6 = ~b; \ - t7 = t1 | t6; \ - e = t3 ^ t7; \ - t9 = a & e; \ - t10 = t1 ^ t6; \ - t11 = t4 & t10; \ - g = t9 ^ t11; \ - t13 = a ^ t3; \ - t14 = t10 & g; \ - f = t13 ^ t14 - -/* 17 terms */ - -#define ib4(a,b,c,d,e,f,g,h) \ - t1 = c ^ d; \ - t2 = c | d; \ - t3 = b ^ t2; \ - t4 = a & t3; \ - f = t1 ^ t4; \ - t6 = a ^ d; \ - t7 = b | d; \ - t8 = t6 & t7; \ - h = t3 ^ t8; \ - t10 = ~a; \ - t11 = c ^ h; \ - t12 = t10 | t11;\ - e = t3 ^ t12; \ - t14 = c | t4; \ - t15 = t7 ^ t14; \ - t16 = h | t10; \ - g = t15 ^ t16 - -/* 16 terms */ - -#define sb5(a,b,c,d,e,f,g,h) \ - t1 = ~a; \ - t2 = a ^ b; \ - t3 = a ^ d; \ - t4 = c ^ t1; \ - t5 = t2 | t3; \ - e = t4 ^ t5; \ - t7 = d & e; \ - t8 = t2 ^ e; \ - t10 = t1 | e; \ - f = t7 ^ t8; \ - t11 = t2 | t7; \ - t12 = t3 ^ t10; \ - t14 = b ^ t7; \ - g = t11 ^ t12; \ - t15 = f & t12; \ - h = t14 ^ t15 - -/* 16 terms */ - -#define ib5(a,b,c,d,e,f,g,h) \ - t1 = ~c; \ - t2 = b & t1; \ - t3 = d ^ t2; \ - t4 = a & t3; \ - t5 = b ^ t1; \ - h = t4 ^ t5; \ - t7 = b | h; \ - t8 = a & t7; \ - f = t3 ^ t8; \ - t10 = a | d; \ - t11 = t1 ^ t7; \ - e = t10 ^ t11; \ - t13 = a ^ c; \ - t14 = b & t10; \ - t15 = t4 | t13; \ - g = t14 ^ t15 - -/* 15 terms */ - -#define sb6(a,b,c,d,e,f,g,h) \ - t1 = ~a; \ - t2 = a ^ d; \ - t3 = b ^ t2; \ - t4 = t1 | t2; \ - t5 = c ^ t4; \ - f = b ^ t5; \ - t13 = ~t5; \ - t7 = t2 | f; \ - t8 = d ^ t7; \ - t9 = t5 & t8; \ - g = t3 ^ t9; \ - t11 = t5 ^ t8; \ - e = g ^ t11; \ - t14 = t3 & t11; \ - h = t13 ^ t14 - -/* 15 terms */ - -#define ib6(a,b,c,d,e,f,g,h) \ - t1 = ~a; \ - t2 = a ^ b; \ - t3 = c ^ t2; \ - t4 = c | t1; \ - t5 = d ^ t4; \ - t13 = d & t1; \ - f = t3 ^ t5; \ - t7 = t3 & t5; \ - t8 = t2 ^ t7; \ - t9 = b | t8; \ - h = t5 ^ t9; \ - t11 = b | h; \ - e = t8 ^ t11; \ - t14 = t3 ^ t11; \ - g = t13 ^ t14 - -/* 17 terms */ - -#define sb7(a,b,c,d,e,f,g,h) \ - t1 = ~c; \ - t2 = b ^ c; \ - t3 = b | t1; \ - t4 = d ^ t3; \ - t5 = a & t4; \ - t7 = a ^ d; \ - h = t2 ^ t5; \ - t8 = b ^ t5; \ - t9 = t2 | t8; \ - t11 = d & t3; \ - f = t7 ^ t9; \ - t12 = t5 ^ f; \ - t15 = t1 | t4; \ - t13 = h & t12; \ - g = t11 ^ t13; \ - t16 = t12 ^ g; \ - e = t15 ^ t16 - -/* 17 terms */ - -#define ib7(a,b,c,d,e,f,g,h) \ - t1 = a & b; \ - t2 = a | b; \ - t3 = c | t1; \ - t4 = d & t2; \ - h = t3 ^ t4; \ - t6 = ~d; \ - t7 = b ^ t4; \ - t8 = h ^ t6; \ - t11 = c ^ t7; \ - t9 = t7 | t8; \ - f = a ^ t9; \ - t12 = d | f; \ - e = t11 ^ t12; \ - t14 = a & h; \ - t15 = t3 ^ f; \ - t16 = e ^ t14; \ - g = t15 ^ t16 - -#define k_xor(r,a,b,c,d) \ - a ^= ks[4 * r + 8]; \ - b ^= ks[4 * r + 9]; \ - c ^= ks[4 * r + 10]; \ - d ^= ks[4 * r + 11] - -#define k_set(r,a,b,c,d) \ - a = ks[4 * r + 8]; \ - b = ks[4 * r + 9]; \ - c = ks[4 * r + 10]; \ - d = ks[4 * r + 11] - -#define k_get(r,a,b,c,d) \ - ks[4 * r + 8] = a; \ - ks[4 * r + 9] = b; \ - ks[4 * r + 10] = c; \ - ks[4 * r + 11] = d - -/* the linear transformation and its inverse */ - -#define rot(a,b,c,d) \ - a = rotl32(a, 13); \ - c = rotl32(c, 3); \ - d ^= c ^ (a << 3); \ - b ^= a ^ c; \ - d = rotl32(d, 7); \ - b = rotl32(b, 1); \ - a ^= b ^ d; \ - c ^= d ^ (b << 7); \ - a = rotl32(a, 5); \ - c = rotl32(c, 22) - -#define irot(a,b,c,d) \ - c = rotr32(c, 22); \ - a = rotr32(a, 5); \ - c ^= d ^ (b << 7); \ - a ^= b ^ d; \ - d = rotr32(d, 7); \ - b = rotr32(b, 1); \ - d ^= c ^ (a << 3); \ - b ^= a ^ c; \ - c = rotr32(c, 3); \ - a = rotr32(a, 13) - -void serpent256_set_key (u32 *ks, const u32 *ukey) -{ - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - ks[i] = ukey[i]; - } - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 132; i++) - { - ks[i + 8] = rotl32 (ks[i + 7] ^ ks[i + 5] ^ ks[i + 3] ^ ks[i + 0] ^ 0x9e3779b9 ^ i, 11); - } - - u32 a,b,c,d,e,f,g,h; - u32 t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16; - - k_set( 0,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get( 0,e,f,g,h); - k_set( 1,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get( 1,e,f,g,h); - k_set( 2,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get( 2,e,f,g,h); - k_set( 3,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get( 3,e,f,g,h); - k_set( 4,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get( 4,e,f,g,h); - k_set( 5,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get( 5,e,f,g,h); - k_set( 6,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get( 6,e,f,g,h); - k_set( 7,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get( 7,e,f,g,h); - k_set( 8,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get( 8,e,f,g,h); - k_set( 9,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get( 9,e,f,g,h); - k_set(10,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(10,e,f,g,h); - k_set(11,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(11,e,f,g,h); - k_set(12,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(12,e,f,g,h); - k_set(13,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(13,e,f,g,h); - k_set(14,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(14,e,f,g,h); - k_set(15,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(15,e,f,g,h); - k_set(16,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(16,e,f,g,h); - k_set(17,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get(17,e,f,g,h); - k_set(18,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(18,e,f,g,h); - k_set(19,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(19,e,f,g,h); - k_set(20,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(20,e,f,g,h); - k_set(21,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(21,e,f,g,h); - k_set(22,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(22,e,f,g,h); - k_set(23,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(23,e,f,g,h); - k_set(24,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(24,e,f,g,h); - k_set(25,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get(25,e,f,g,h); - k_set(26,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(26,e,f,g,h); - k_set(27,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(27,e,f,g,h); - k_set(28,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(28,e,f,g,h); - k_set(29,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(29,e,f,g,h); - k_set(30,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(30,e,f,g,h); - k_set(31,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(31,e,f,g,h); - k_set(32,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(32,e,f,g,h); -} - -void serpent256_encrypt (const u32 *ks, const u32 *in, u32 *out) -{ - u32 a,b,c,d,e,f,g,h; - u32 t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16; - - a = in[0]; - b = in[1]; - c = in[2]; - d = in[3]; - - k_xor( 0,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor( 1,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor( 2,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor( 3,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor( 4,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor( 5,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor( 6,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor( 7,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor( 8,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor( 9,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(10,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(11,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(12,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(13,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(14,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(15,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(16,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(17,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(18,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(19,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(20,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(21,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(22,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(23,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(24,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(25,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(26,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(27,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(28,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(29,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d); - k_xor(30,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h); - k_xor(31,e,f,g,h); sb7(e,f,g,h,a,b,c,d); - k_xor(32,a,b,c,d); - - out[0] = a; - out[1] = b; - out[2] = c; - out[3] = d; -} - -void serpent256_decrypt (const u32 *ks, const u32 *in, u32 *out) -{ - u32 a,b,c,d,e,f,g,h; - u32 t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16; - - a = in[0]; - b = in[1]; - c = in[2]; - d = in[3]; - - k_xor(32,a,b,c,d); - ib7(a,b,c,d,e,f,g,h); k_xor(31,e,f,g,h); - irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(30,a,b,c,d); - irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(29,e,f,g,h); - irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(28,a,b,c,d); - irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(27,e,f,g,h); - irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(26,a,b,c,d); - irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor(25,e,f,g,h); - irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor(24,a,b,c,d); - irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor(23,e,f,g,h); - irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(22,a,b,c,d); - irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(21,e,f,g,h); - irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(20,a,b,c,d); - irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(19,e,f,g,h); - irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(18,a,b,c,d); - irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor(17,e,f,g,h); - irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor(16,a,b,c,d); - irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor(15,e,f,g,h); - irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(14,a,b,c,d); - irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(13,e,f,g,h); - irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(12,a,b,c,d); - irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(11,e,f,g,h); - irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(10,a,b,c,d); - irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor( 9,e,f,g,h); - irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor( 8,a,b,c,d); - irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor( 7,e,f,g,h); - irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor( 6,a,b,c,d); - irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor( 5,e,f,g,h); - irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor( 4,a,b,c,d); - irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor( 3,e,f,g,h); - irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor( 2,a,b,c,d); - irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor( 1,e,f,g,h); - irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor( 0,a,b,c,d); - - out[0] = a; - out[1] = b; - out[2] = c; - out[3] = d; -} - -void serpent256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out) -{ - u32 T[4] = { 0 }; - u32 Z[4] = { 0 }; - - out[0] = in[0]; - out[1] = in[1]; - out[2] = in[2]; - out[3] = in[3]; - - u32 ks[140]; - - serpent256_set_key (ks, ukey2); - serpent256_encrypt (ks, Z, T); - - out[0] ^= T[0]; - out[1] ^= T[1]; - out[2] ^= T[2]; - out[3] ^= T[3]; - - serpent256_set_key (ks, ukey1); - serpent256_decrypt (ks, out, out); - - out[0] ^= T[0]; - out[1] ^= T[1]; - out[2] ^= T[2]; - out[3] ^= T[3]; -} diff --git a/OpenCL/kernel_twofish256.c b/OpenCL/kernel_twofish256.c deleted file mode 100644 index 6482a63..0000000 --- a/OpenCL/kernel_twofish256.c +++ /dev/null @@ -1,466 +0,0 @@ -/* This is an independent implementation of the encryption algorithm: */ -/* */ -/* Twofish by Bruce Schneier and colleagues */ -/* */ -/* which is a candidate algorithm in the Advanced Encryption Standard */ -/* programme of the US National Institute of Standards and Technology. */ -/* */ -/* Copyright in this implementation is held by Dr B R Gladman but I */ -/* hereby give permission for its free direct or derivative use subject */ -/* to acknowledgment of its origin and compliance with any conditions */ -/* that the originators of t he algorithm place on its exploitation. */ -/* */ -/* My thanks to Doug Whiting and Niels Ferguson for comments that led */ -/* to improvements in this implementation. */ -/* */ -/* Dr Brian Gladman (gladman@seven77.demon.co.uk) 14th January 1999 */ -/* */ -/* -------------------------------------------------------------------- */ -/* */ -/* Cleaned and optimized for GPU use with hashcat by Jens Steube */ - -#define extract_byte(x,n) (((x) >> (8 * (n))) & 0xff) - -__constant u32 q_tab[2][256] = -{ - { - 0xA9, 0x67, 0xB3, 0xE8, 0x04, 0xFD, 0xA3, 0x76, 0x9A, 0x92, 0x80, 0x78, - 0xE4, 0xDD, 0xD1, 0x38, 0x0D, 0xC6, 0x35, 0x98, 0x18, 0xF7, 0xEC, 0x6C, - 0x43, 0x75, 0x37, 0x26, 0xFA, 0x13, 0x94, 0x48, 0xF2, 0xD0, 0x8B, 0x30, - 0x84, 0x54, 0xDF, 0x23, 0x19, 0x5B, 0x3D, 0x59, 0xF3, 0xAE, 0xA2, 0x82, - 0x63, 0x01, 0x83, 0x2E, 0xD9, 0x51, 0x9B, 0x7C, 0xA6, 0xEB, 0xA5, 0xBE, - 0x16, 0x0C, 0xE3, 0x61, 0xC0, 0x8C, 0x3A, 0xF5, 0x73, 0x2C, 0x25, 0x0B, - 0xBB, 0x4E, 0x89, 0x6B, 0x53, 0x6A, 0xB4, 0xF1, 0xE1, 0xE6, 0xBD, 0x45, - 0xE2, 0xF4, 0xB6, 0x66, 0xCC, 0x95, 0x03, 0x56, 0xD4, 0x1C, 0x1E, 0xD7, - 0xFB, 0xC3, 0x8E, 0xB5, 0xE9, 0xCF, 0xBF, 0xBA, 0xEA, 0x77, 0x39, 0xAF, - 0x33, 0xC9, 0x62, 0x71, 0x81, 0x79, 0x09, 0xAD, 0x24, 0xCD, 0xF9, 0xD8, - 0xE5, 0xC5, 0xB9, 0x4D, 0x44, 0x08, 0x86, 0xE7, 0xA1, 0x1D, 0xAA, 0xED, - 0x06, 0x70, 0xB2, 0xD2, 0x41, 0x7B, 0xA0, 0x11, 0x31, 0xC2, 0x27, 0x90, - 0x20, 0xF6, 0x60, 0xFF, 0x96, 0x5C, 0xB1, 0xAB, 0x9E, 0x9C, 0x52, 0x1B, - 0x5F, 0x93, 0x0A, 0xEF, 0x91, 0x85, 0x49, 0xEE, 0x2D, 0x4F, 0x8F, 0x3B, - 0x47, 0x87, 0x6D, 0x46, 0xD6, 0x3E, 0x69, 0x64, 0x2A, 0xCE, 0xCB, 0x2F, - 0xFC, 0x97, 0x05, 0x7A, 0xAC, 0x7F, 0xD5, 0x1A, 0x4B, 0x0E, 0xA7, 0x5A, - 0x28, 0x14, 0x3F, 0x29, 0x88, 0x3C, 0x4C, 0x02, 0xB8, 0xDA, 0xB0, 0x17, - 0x55, 0x1F, 0x8A, 0x7D, 0x57, 0xC7, 0x8D, 0x74, 0xB7, 0xC4, 0x9F, 0x72, - 0x7E, 0x15, 0x22, 0x12, 0x58, 0x07, 0x99, 0x34, 0x6E, 0x50, 0xDE, 0x68, - 0x65, 0xBC, 0xDB, 0xF8, 0xC8, 0xA8, 0x2B, 0x40, 0xDC, 0xFE, 0x32, 0xA4, - 0xCA, 0x10, 0x21, 0xF0, 0xD3, 0x5D, 0x0F, 0x00, 0x6F, 0x9D, 0x36, 0x42, - 0x4A, 0x5E, 0xC1, 0xE0 - }, - { - 0x75, 0xF3, 0xC6, 0xF4, 0xDB, 0x7B, 0xFB, 0xC8, 0x4A, 0xD3, 0xE6, 0x6B, - 0x45, 0x7D, 0xE8, 0x4B, 0xD6, 0x32, 0xD8, 0xFD, 0x37, 0x71, 0xF1, 0xE1, - 0x30, 0x0F, 0xF8, 0x1B, 0x87, 0xFA, 0x06, 0x3F, 0x5E, 0xBA, 0xAE, 0x5B, - 0x8A, 0x00, 0xBC, 0x9D, 0x6D, 0xC1, 0xB1, 0x0E, 0x80, 0x5D, 0xD2, 0xD5, - 0xA0, 0x84, 0x07, 0x14, 0xB5, 0x90, 0x2C, 0xA3, 0xB2, 0x73, 0x4C, 0x54, - 0x92, 0x74, 0x36, 0x51, 0x38, 0xB0, 0xBD, 0x5A, 0xFC, 0x60, 0x62, 0x96, - 0x6C, 0x42, 0xF7, 0x10, 0x7C, 0x28, 0x27, 0x8C, 0x13, 0x95, 0x9C, 0xC7, - 0x24, 0x46, 0x3B, 0x70, 0xCA, 0xE3, 0x85, 0xCB, 0x11, 0xD0, 0x93, 0xB8, - 0xA6, 0x83, 0x20, 0xFF, 0x9F, 0x77, 0xC3, 0xCC, 0x03, 0x6F, 0x08, 0xBF, - 0x40, 0xE7, 0x2B, 0xE2, 0x79, 0x0C, 0xAA, 0x82, 0x41, 0x3A, 0xEA, 0xB9, - 0xE4, 0x9A, 0xA4, 0x97, 0x7E, 0xDA, 0x7A, 0x17, 0x66, 0x94, 0xA1, 0x1D, - 0x3D, 0xF0, 0xDE, 0xB3, 0x0B, 0x72, 0xA7, 0x1C, 0xEF, 0xD1, 0x53, 0x3E, - 0x8F, 0x33, 0x26, 0x5F, 0xEC, 0x76, 0x2A, 0x49, 0x81, 0x88, 0xEE, 0x21, - 0xC4, 0x1A, 0xEB, 0xD9, 0xC5, 0x39, 0x99, 0xCD, 0xAD, 0x31, 0x8B, 0x01, - 0x18, 0x23, 0xDD, 0x1F, 0x4E, 0x2D, 0xF9, 0x48, 0x4F, 0xF2, 0x65, 0x8E, - 0x78, 0x5C, 0x58, 0x19, 0x8D, 0xE5, 0x98, 0x57, 0x67, 0x7F, 0x05, 0x64, - 0xAF, 0x63, 0xB6, 0xFE, 0xF5, 0xB7, 0x3C, 0xA5, 0xCE, 0xE9, 0x68, 0x44, - 0xE0, 0x4D, 0x43, 0x69, 0x29, 0x2E, 0xAC, 0x15, 0x59, 0xA8, 0x0A, 0x9E, - 0x6E, 0x47, 0xDF, 0x34, 0x35, 0x6A, 0xCF, 0xDC, 0x22, 0xC9, 0xC0, 0x9B, - 0x89, 0xD4, 0xED, 0xAB, 0x12, 0xA2, 0x0D, 0x52, 0xBB, 0x02, 0x2F, 0xA9, - 0xD7, 0x61, 0x1E, 0xB4, 0x50, 0x04, 0xF6, 0xC2, 0x16, 0x25, 0x86, 0x56, - 0x55, 0x09, 0xBE, 0x91 - } -}; - -#define q(n,x) q_tab[n][x] - -__constant u32 m_tab[4][256] = -{ - { 0xBCBC3275, 0xECEC21F3, 0x202043C6, 0xB3B3C9F4, 0xDADA03DB, 0x02028B7B, - 0xE2E22BFB, 0x9E9EFAC8, 0xC9C9EC4A, 0xD4D409D3, 0x18186BE6, 0x1E1E9F6B, - 0x98980E45, 0xB2B2387D, 0xA6A6D2E8, 0x2626B74B, 0x3C3C57D6, 0x93938A32, - 0x8282EED8, 0x525298FD, 0x7B7BD437, 0xBBBB3771, 0x5B5B97F1, 0x474783E1, - 0x24243C30, 0x5151E20F, 0xBABAC6F8, 0x4A4AF31B, 0xBFBF4887, 0x0D0D70FA, - 0xB0B0B306, 0x7575DE3F, 0xD2D2FD5E, 0x7D7D20BA, 0x666631AE, 0x3A3AA35B, - 0x59591C8A, 0x00000000, 0xCDCD93BC, 0x1A1AE09D, 0xAEAE2C6D, 0x7F7FABC1, - 0x2B2BC7B1, 0xBEBEB90E, 0xE0E0A080, 0x8A8A105D, 0x3B3B52D2, 0x6464BAD5, - 0xD8D888A0, 0xE7E7A584, 0x5F5FE807, 0x1B1B1114, 0x2C2CC2B5, 0xFCFCB490, - 0x3131272C, 0x808065A3, 0x73732AB2, 0x0C0C8173, 0x79795F4C, 0x6B6B4154, - 0x4B4B0292, 0x53536974, 0x94948F36, 0x83831F51, 0x2A2A3638, 0xC4C49CB0, - 0x2222C8BD, 0xD5D5F85A, 0xBDBDC3FC, 0x48487860, 0xFFFFCE62, 0x4C4C0796, - 0x4141776C, 0xC7C7E642, 0xEBEB24F7, 0x1C1C1410, 0x5D5D637C, 0x36362228, - 0x6767C027, 0xE9E9AF8C, 0x4444F913, 0x1414EA95, 0xF5F5BB9C, 0xCFCF18C7, - 0x3F3F2D24, 0xC0C0E346, 0x7272DB3B, 0x54546C70, 0x29294CCA, 0xF0F035E3, - 0x0808FE85, 0xC6C617CB, 0xF3F34F11, 0x8C8CE4D0, 0xA4A45993, 0xCACA96B8, - 0x68683BA6, 0xB8B84D83, 0x38382820, 0xE5E52EFF, 0xADAD569F, 0x0B0B8477, - 0xC8C81DC3, 0x9999FFCC, 0x5858ED03, 0x19199A6F, 0x0E0E0A08, 0x95957EBF, - 0x70705040, 0xF7F730E7, 0x6E6ECF2B, 0x1F1F6EE2, 0xB5B53D79, 0x09090F0C, - 0x616134AA, 0x57571682, 0x9F9F0B41, 0x9D9D803A, 0x111164EA, 0x2525CDB9, - 0xAFAFDDE4, 0x4545089A, 0xDFDF8DA4, 0xA3A35C97, 0xEAEAD57E, 0x353558DA, - 0xEDEDD07A, 0x4343FC17, 0xF8F8CB66, 0xFBFBB194, 0x3737D3A1, 0xFAFA401D, - 0xC2C2683D, 0xB4B4CCF0, 0x32325DDE, 0x9C9C71B3, 0x5656E70B, 0xE3E3DA72, - 0x878760A7, 0x15151B1C, 0xF9F93AEF, 0x6363BFD1, 0x3434A953, 0x9A9A853E, - 0xB1B1428F, 0x7C7CD133, 0x88889B26, 0x3D3DA65F, 0xA1A1D7EC, 0xE4E4DF76, - 0x8181942A, 0x91910149, 0x0F0FFB81, 0xEEEEAA88, 0x161661EE, 0xD7D77321, - 0x9797F5C4, 0xA5A5A81A, 0xFEFE3FEB, 0x6D6DB5D9, 0x7878AEC5, 0xC5C56D39, - 0x1D1DE599, 0x7676A4CD, 0x3E3EDCAD, 0xCBCB6731, 0xB6B6478B, 0xEFEF5B01, - 0x12121E18, 0x6060C523, 0x6A6AB0DD, 0x4D4DF61F, 0xCECEE94E, 0xDEDE7C2D, - 0x55559DF9, 0x7E7E5A48, 0x2121B24F, 0x03037AF2, 0xA0A02665, 0x5E5E198E, - 0x5A5A6678, 0x65654B5C, 0x62624E58, 0xFDFD4519, 0x0606F48D, 0x404086E5, - 0xF2F2BE98, 0x3333AC57, 0x17179067, 0x05058E7F, 0xE8E85E05, 0x4F4F7D64, - 0x89896AAF, 0x10109563, 0x74742FB6, 0x0A0A75FE, 0x5C5C92F5, 0x9B9B74B7, - 0x2D2D333C, 0x3030D6A5, 0x2E2E49CE, 0x494989E9, 0x46467268, 0x77775544, - 0xA8A8D8E0, 0x9696044D, 0x2828BD43, 0xA9A92969, 0xD9D97929, 0x8686912E, - 0xD1D187AC, 0xF4F44A15, 0x8D8D1559, 0xD6D682A8, 0xB9B9BC0A, 0x42420D9E, - 0xF6F6C16E, 0x2F2FB847, 0xDDDD06DF, 0x23233934, 0xCCCC6235, 0xF1F1C46A, - 0xC1C112CF, 0x8585EBDC, 0x8F8F9E22, 0x7171A1C9, 0x9090F0C0, 0xAAAA539B, - 0x0101F189, 0x8B8BE1D4, 0x4E4E8CED, 0x8E8E6FAB, 0xABABA212, 0x6F6F3EA2, - 0xE6E6540D, 0xDBDBF252, 0x92927BBB, 0xB7B7B602, 0x6969CA2F, 0x3939D9A9, - 0xD3D30CD7, 0xA7A72361, 0xA2A2AD1E, 0xC3C399B4, 0x6C6C4450, 0x07070504, - 0x04047FF6, 0x272746C2, 0xACACA716, 0xD0D07625, 0x50501386, 0xDCDCF756, - 0x84841A55, 0xE1E15109, 0x7A7A25BE, 0x1313EF91 }, - - { 0xA9D93939, 0x67901717, 0xB3719C9C, 0xE8D2A6A6, 0x04050707, 0xFD985252, - 0xA3658080, 0x76DFE4E4, 0x9A084545, 0x92024B4B, 0x80A0E0E0, 0x78665A5A, - 0xE4DDAFAF, 0xDDB06A6A, 0xD1BF6363, 0x38362A2A, 0x0D54E6E6, 0xC6432020, - 0x3562CCCC, 0x98BEF2F2, 0x181E1212, 0xF724EBEB, 0xECD7A1A1, 0x6C774141, - 0x43BD2828, 0x7532BCBC, 0x37D47B7B, 0x269B8888, 0xFA700D0D, 0x13F94444, - 0x94B1FBFB, 0x485A7E7E, 0xF27A0303, 0xD0E48C8C, 0x8B47B6B6, 0x303C2424, - 0x84A5E7E7, 0x54416B6B, 0xDF06DDDD, 0x23C56060, 0x1945FDFD, 0x5BA33A3A, - 0x3D68C2C2, 0x59158D8D, 0xF321ECEC, 0xAE316666, 0xA23E6F6F, 0x82165757, - 0x63951010, 0x015BEFEF, 0x834DB8B8, 0x2E918686, 0xD9B56D6D, 0x511F8383, - 0x9B53AAAA, 0x7C635D5D, 0xA63B6868, 0xEB3FFEFE, 0xA5D63030, 0xBE257A7A, - 0x16A7ACAC, 0x0C0F0909, 0xE335F0F0, 0x6123A7A7, 0xC0F09090, 0x8CAFE9E9, - 0x3A809D9D, 0xF5925C5C, 0x73810C0C, 0x2C273131, 0x2576D0D0, 0x0BE75656, - 0xBB7B9292, 0x4EE9CECE, 0x89F10101, 0x6B9F1E1E, 0x53A93434, 0x6AC4F1F1, - 0xB499C3C3, 0xF1975B5B, 0xE1834747, 0xE66B1818, 0xBDC82222, 0x450E9898, - 0xE26E1F1F, 0xF4C9B3B3, 0xB62F7474, 0x66CBF8F8, 0xCCFF9999, 0x95EA1414, - 0x03ED5858, 0x56F7DCDC, 0xD4E18B8B, 0x1C1B1515, 0x1EADA2A2, 0xD70CD3D3, - 0xFB2BE2E2, 0xC31DC8C8, 0x8E195E5E, 0xB5C22C2C, 0xE9894949, 0xCF12C1C1, - 0xBF7E9595, 0xBA207D7D, 0xEA641111, 0x77840B0B, 0x396DC5C5, 0xAF6A8989, - 0x33D17C7C, 0xC9A17171, 0x62CEFFFF, 0x7137BBBB, 0x81FB0F0F, 0x793DB5B5, - 0x0951E1E1, 0xADDC3E3E, 0x242D3F3F, 0xCDA47676, 0xF99D5555, 0xD8EE8282, - 0xE5864040, 0xC5AE7878, 0xB9CD2525, 0x4D049696, 0x44557777, 0x080A0E0E, - 0x86135050, 0xE730F7F7, 0xA1D33737, 0x1D40FAFA, 0xAA346161, 0xED8C4E4E, - 0x06B3B0B0, 0x706C5454, 0xB22A7373, 0xD2523B3B, 0x410B9F9F, 0x7B8B0202, - 0xA088D8D8, 0x114FF3F3, 0x3167CBCB, 0xC2462727, 0x27C06767, 0x90B4FCFC, - 0x20283838, 0xF67F0404, 0x60784848, 0xFF2EE5E5, 0x96074C4C, 0x5C4B6565, - 0xB1C72B2B, 0xAB6F8E8E, 0x9E0D4242, 0x9CBBF5F5, 0x52F2DBDB, 0x1BF34A4A, - 0x5FA63D3D, 0x9359A4A4, 0x0ABCB9B9, 0xEF3AF9F9, 0x91EF1313, 0x85FE0808, - 0x49019191, 0xEE611616, 0x2D7CDEDE, 0x4FB22121, 0x8F42B1B1, 0x3BDB7272, - 0x47B82F2F, 0x8748BFBF, 0x6D2CAEAE, 0x46E3C0C0, 0xD6573C3C, 0x3E859A9A, - 0x6929A9A9, 0x647D4F4F, 0x2A948181, 0xCE492E2E, 0xCB17C6C6, 0x2FCA6969, - 0xFCC3BDBD, 0x975CA3A3, 0x055EE8E8, 0x7AD0EDED, 0xAC87D1D1, 0x7F8E0505, - 0xD5BA6464, 0x1AA8A5A5, 0x4BB72626, 0x0EB9BEBE, 0xA7608787, 0x5AF8D5D5, - 0x28223636, 0x14111B1B, 0x3FDE7575, 0x2979D9D9, 0x88AAEEEE, 0x3C332D2D, - 0x4C5F7979, 0x02B6B7B7, 0xB896CACA, 0xDA583535, 0xB09CC4C4, 0x17FC4343, - 0x551A8484, 0x1FF64D4D, 0x8A1C5959, 0x7D38B2B2, 0x57AC3333, 0xC718CFCF, - 0x8DF40606, 0x74695353, 0xB7749B9B, 0xC4F59797, 0x9F56ADAD, 0x72DAE3E3, - 0x7ED5EAEA, 0x154AF4F4, 0x229E8F8F, 0x12A2ABAB, 0x584E6262, 0x07E85F5F, - 0x99E51D1D, 0x34392323, 0x6EC1F6F6, 0x50446C6C, 0xDE5D3232, 0x68724646, - 0x6526A0A0, 0xBC93CDCD, 0xDB03DADA, 0xF8C6BABA, 0xC8FA9E9E, 0xA882D6D6, - 0x2BCF6E6E, 0x40507070, 0xDCEB8585, 0xFE750A0A, 0x328A9393, 0xA48DDFDF, - 0xCA4C2929, 0x10141C1C, 0x2173D7D7, 0xF0CCB4B4, 0xD309D4D4, 0x5D108A8A, - 0x0FE25151, 0x00000000, 0x6F9A1919, 0x9DE01A1A, 0x368F9494, 0x42E6C7C7, - 0x4AECC9C9, 0x5EFDD2D2, 0xC1AB7F7F, 0xE0D8A8A8 }, - - { 0xBC75BC32, 0xECF3EC21, 0x20C62043, 0xB3F4B3C9, 0xDADBDA03, 0x027B028B, - 0xE2FBE22B, 0x9EC89EFA, 0xC94AC9EC, 0xD4D3D409, 0x18E6186B, 0x1E6B1E9F, - 0x9845980E, 0xB27DB238, 0xA6E8A6D2, 0x264B26B7, 0x3CD63C57, 0x9332938A, - 0x82D882EE, 0x52FD5298, 0x7B377BD4, 0xBB71BB37, 0x5BF15B97, 0x47E14783, - 0x2430243C, 0x510F51E2, 0xBAF8BAC6, 0x4A1B4AF3, 0xBF87BF48, 0x0DFA0D70, - 0xB006B0B3, 0x753F75DE, 0xD25ED2FD, 0x7DBA7D20, 0x66AE6631, 0x3A5B3AA3, - 0x598A591C, 0x00000000, 0xCDBCCD93, 0x1A9D1AE0, 0xAE6DAE2C, 0x7FC17FAB, - 0x2BB12BC7, 0xBE0EBEB9, 0xE080E0A0, 0x8A5D8A10, 0x3BD23B52, 0x64D564BA, - 0xD8A0D888, 0xE784E7A5, 0x5F075FE8, 0x1B141B11, 0x2CB52CC2, 0xFC90FCB4, - 0x312C3127, 0x80A38065, 0x73B2732A, 0x0C730C81, 0x794C795F, 0x6B546B41, - 0x4B924B02, 0x53745369, 0x9436948F, 0x8351831F, 0x2A382A36, 0xC4B0C49C, - 0x22BD22C8, 0xD55AD5F8, 0xBDFCBDC3, 0x48604878, 0xFF62FFCE, 0x4C964C07, - 0x416C4177, 0xC742C7E6, 0xEBF7EB24, 0x1C101C14, 0x5D7C5D63, 0x36283622, - 0x672767C0, 0xE98CE9AF, 0x441344F9, 0x149514EA, 0xF59CF5BB, 0xCFC7CF18, - 0x3F243F2D, 0xC046C0E3, 0x723B72DB, 0x5470546C, 0x29CA294C, 0xF0E3F035, - 0x088508FE, 0xC6CBC617, 0xF311F34F, 0x8CD08CE4, 0xA493A459, 0xCAB8CA96, - 0x68A6683B, 0xB883B84D, 0x38203828, 0xE5FFE52E, 0xAD9FAD56, 0x0B770B84, - 0xC8C3C81D, 0x99CC99FF, 0x580358ED, 0x196F199A, 0x0E080E0A, 0x95BF957E, - 0x70407050, 0xF7E7F730, 0x6E2B6ECF, 0x1FE21F6E, 0xB579B53D, 0x090C090F, - 0x61AA6134, 0x57825716, 0x9F419F0B, 0x9D3A9D80, 0x11EA1164, 0x25B925CD, - 0xAFE4AFDD, 0x459A4508, 0xDFA4DF8D, 0xA397A35C, 0xEA7EEAD5, 0x35DA3558, - 0xED7AEDD0, 0x431743FC, 0xF866F8CB, 0xFB94FBB1, 0x37A137D3, 0xFA1DFA40, - 0xC23DC268, 0xB4F0B4CC, 0x32DE325D, 0x9CB39C71, 0x560B56E7, 0xE372E3DA, - 0x87A78760, 0x151C151B, 0xF9EFF93A, 0x63D163BF, 0x345334A9, 0x9A3E9A85, - 0xB18FB142, 0x7C337CD1, 0x8826889B, 0x3D5F3DA6, 0xA1ECA1D7, 0xE476E4DF, - 0x812A8194, 0x91499101, 0x0F810FFB, 0xEE88EEAA, 0x16EE1661, 0xD721D773, - 0x97C497F5, 0xA51AA5A8, 0xFEEBFE3F, 0x6DD96DB5, 0x78C578AE, 0xC539C56D, - 0x1D991DE5, 0x76CD76A4, 0x3EAD3EDC, 0xCB31CB67, 0xB68BB647, 0xEF01EF5B, - 0x1218121E, 0x602360C5, 0x6ADD6AB0, 0x4D1F4DF6, 0xCE4ECEE9, 0xDE2DDE7C, - 0x55F9559D, 0x7E487E5A, 0x214F21B2, 0x03F2037A, 0xA065A026, 0x5E8E5E19, - 0x5A785A66, 0x655C654B, 0x6258624E, 0xFD19FD45, 0x068D06F4, 0x40E54086, - 0xF298F2BE, 0x335733AC, 0x17671790, 0x057F058E, 0xE805E85E, 0x4F644F7D, - 0x89AF896A, 0x10631095, 0x74B6742F, 0x0AFE0A75, 0x5CF55C92, 0x9BB79B74, - 0x2D3C2D33, 0x30A530D6, 0x2ECE2E49, 0x49E94989, 0x46684672, 0x77447755, - 0xA8E0A8D8, 0x964D9604, 0x284328BD, 0xA969A929, 0xD929D979, 0x862E8691, - 0xD1ACD187, 0xF415F44A, 0x8D598D15, 0xD6A8D682, 0xB90AB9BC, 0x429E420D, - 0xF66EF6C1, 0x2F472FB8, 0xDDDFDD06, 0x23342339, 0xCC35CC62, 0xF16AF1C4, - 0xC1CFC112, 0x85DC85EB, 0x8F228F9E, 0x71C971A1, 0x90C090F0, 0xAA9BAA53, - 0x018901F1, 0x8BD48BE1, 0x4EED4E8C, 0x8EAB8E6F, 0xAB12ABA2, 0x6FA26F3E, - 0xE60DE654, 0xDB52DBF2, 0x92BB927B, 0xB702B7B6, 0x692F69CA, 0x39A939D9, - 0xD3D7D30C, 0xA761A723, 0xA21EA2AD, 0xC3B4C399, 0x6C506C44, 0x07040705, - 0x04F6047F, 0x27C22746, 0xAC16ACA7, 0xD025D076, 0x50865013, 0xDC56DCF7, - 0x8455841A, 0xE109E151, 0x7ABE7A25, 0x139113EF }, - - { 0xD939A9D9, 0x90176790, 0x719CB371, 0xD2A6E8D2, 0x05070405, 0x9852FD98, - 0x6580A365, 0xDFE476DF, 0x08459A08, 0x024B9202, 0xA0E080A0, 0x665A7866, - 0xDDAFE4DD, 0xB06ADDB0, 0xBF63D1BF, 0x362A3836, 0x54E60D54, 0x4320C643, - 0x62CC3562, 0xBEF298BE, 0x1E12181E, 0x24EBF724, 0xD7A1ECD7, 0x77416C77, - 0xBD2843BD, 0x32BC7532, 0xD47B37D4, 0x9B88269B, 0x700DFA70, 0xF94413F9, - 0xB1FB94B1, 0x5A7E485A, 0x7A03F27A, 0xE48CD0E4, 0x47B68B47, 0x3C24303C, - 0xA5E784A5, 0x416B5441, 0x06DDDF06, 0xC56023C5, 0x45FD1945, 0xA33A5BA3, - 0x68C23D68, 0x158D5915, 0x21ECF321, 0x3166AE31, 0x3E6FA23E, 0x16578216, - 0x95106395, 0x5BEF015B, 0x4DB8834D, 0x91862E91, 0xB56DD9B5, 0x1F83511F, - 0x53AA9B53, 0x635D7C63, 0x3B68A63B, 0x3FFEEB3F, 0xD630A5D6, 0x257ABE25, - 0xA7AC16A7, 0x0F090C0F, 0x35F0E335, 0x23A76123, 0xF090C0F0, 0xAFE98CAF, - 0x809D3A80, 0x925CF592, 0x810C7381, 0x27312C27, 0x76D02576, 0xE7560BE7, - 0x7B92BB7B, 0xE9CE4EE9, 0xF10189F1, 0x9F1E6B9F, 0xA93453A9, 0xC4F16AC4, - 0x99C3B499, 0x975BF197, 0x8347E183, 0x6B18E66B, 0xC822BDC8, 0x0E98450E, - 0x6E1FE26E, 0xC9B3F4C9, 0x2F74B62F, 0xCBF866CB, 0xFF99CCFF, 0xEA1495EA, - 0xED5803ED, 0xF7DC56F7, 0xE18BD4E1, 0x1B151C1B, 0xADA21EAD, 0x0CD3D70C, - 0x2BE2FB2B, 0x1DC8C31D, 0x195E8E19, 0xC22CB5C2, 0x8949E989, 0x12C1CF12, - 0x7E95BF7E, 0x207DBA20, 0x6411EA64, 0x840B7784, 0x6DC5396D, 0x6A89AF6A, - 0xD17C33D1, 0xA171C9A1, 0xCEFF62CE, 0x37BB7137, 0xFB0F81FB, 0x3DB5793D, - 0x51E10951, 0xDC3EADDC, 0x2D3F242D, 0xA476CDA4, 0x9D55F99D, 0xEE82D8EE, - 0x8640E586, 0xAE78C5AE, 0xCD25B9CD, 0x04964D04, 0x55774455, 0x0A0E080A, - 0x13508613, 0x30F7E730, 0xD337A1D3, 0x40FA1D40, 0x3461AA34, 0x8C4EED8C, - 0xB3B006B3, 0x6C54706C, 0x2A73B22A, 0x523BD252, 0x0B9F410B, 0x8B027B8B, - 0x88D8A088, 0x4FF3114F, 0x67CB3167, 0x4627C246, 0xC06727C0, 0xB4FC90B4, - 0x28382028, 0x7F04F67F, 0x78486078, 0x2EE5FF2E, 0x074C9607, 0x4B655C4B, - 0xC72BB1C7, 0x6F8EAB6F, 0x0D429E0D, 0xBBF59CBB, 0xF2DB52F2, 0xF34A1BF3, - 0xA63D5FA6, 0x59A49359, 0xBCB90ABC, 0x3AF9EF3A, 0xEF1391EF, 0xFE0885FE, - 0x01914901, 0x6116EE61, 0x7CDE2D7C, 0xB2214FB2, 0x42B18F42, 0xDB723BDB, - 0xB82F47B8, 0x48BF8748, 0x2CAE6D2C, 0xE3C046E3, 0x573CD657, 0x859A3E85, - 0x29A96929, 0x7D4F647D, 0x94812A94, 0x492ECE49, 0x17C6CB17, 0xCA692FCA, - 0xC3BDFCC3, 0x5CA3975C, 0x5EE8055E, 0xD0ED7AD0, 0x87D1AC87, 0x8E057F8E, - 0xBA64D5BA, 0xA8A51AA8, 0xB7264BB7, 0xB9BE0EB9, 0x6087A760, 0xF8D55AF8, - 0x22362822, 0x111B1411, 0xDE753FDE, 0x79D92979, 0xAAEE88AA, 0x332D3C33, - 0x5F794C5F, 0xB6B702B6, 0x96CAB896, 0x5835DA58, 0x9CC4B09C, 0xFC4317FC, - 0x1A84551A, 0xF64D1FF6, 0x1C598A1C, 0x38B27D38, 0xAC3357AC, 0x18CFC718, - 0xF4068DF4, 0x69537469, 0x749BB774, 0xF597C4F5, 0x56AD9F56, 0xDAE372DA, - 0xD5EA7ED5, 0x4AF4154A, 0x9E8F229E, 0xA2AB12A2, 0x4E62584E, 0xE85F07E8, - 0xE51D99E5, 0x39233439, 0xC1F66EC1, 0x446C5044, 0x5D32DE5D, 0x72466872, - 0x26A06526, 0x93CDBC93, 0x03DADB03, 0xC6BAF8C6, 0xFA9EC8FA, 0x82D6A882, - 0xCF6E2BCF, 0x50704050, 0xEB85DCEB, 0x750AFE75, 0x8A93328A, 0x8DDFA48D, - 0x4C29CA4C, 0x141C1014, 0x73D72173, 0xCCB4F0CC, 0x09D4D309, 0x108A5D10, - 0xE2510FE2, 0x00000000, 0x9A196F9A, 0xE01A9DE0, 0x8F94368F, 0xE6C742E6, - 0xECC94AEC, 0xFDD25EFD, 0xAB7FC1AB, 0xD8A8E0D8 } -}; - -#define mds(n,x) m_tab[n][x] - -u32 h_fun (u32 *sk, u32 *lk, const u32 x, const u32 *key) -{ - u32 b0, b1, b2, b3; - - b0 = extract_byte (x, 0); - b1 = extract_byte (x, 1); - b2 = extract_byte (x, 2); - b3 = extract_byte (x, 3); - - b0 = q (1, b0) ^ extract_byte (key[3], 0); - b1 = q (0, b1) ^ extract_byte (key[3], 1); - b2 = q (0, b2) ^ extract_byte (key[3], 2); - b3 = q (1, b3) ^ extract_byte (key[3], 3); - - b0 = q (1, b0) ^ extract_byte (key[2], 0); - b1 = q (1, b1) ^ extract_byte (key[2], 1); - b2 = q (0, b2) ^ extract_byte (key[2], 2); - b3 = q (0, b3) ^ extract_byte (key[2], 3); - - b0 = q (0, (q (0, b0) ^ extract_byte (key[1], 0))) ^ extract_byte (key[0], 0); - b1 = q (0, (q (1, b1) ^ extract_byte (key[1], 1))) ^ extract_byte (key[0], 1); - b2 = q (1, (q (0, b2) ^ extract_byte (key[1], 2))) ^ extract_byte (key[0], 2); - b3 = q (1, (q (1, b3) ^ extract_byte (key[1], 3))) ^ extract_byte (key[0], 3); - - return mds (0, b0) ^ mds (1, b1) ^ mds (2, b2) ^ mds (3, b3); -} - -#define q40(x,k) q (0, q (0, q (1, q (1, x) ^ extract_byte (k[3], 0)) ^ extract_byte (k[2], 0)) ^ extract_byte (k[1], 0)) ^ extract_byte (k[0], 0) -#define q41(x,k) q (0, q (1, q (1, q (0, x) ^ extract_byte (k[3], 1)) ^ extract_byte (k[2], 1)) ^ extract_byte (k[1], 1)) ^ extract_byte (k[0], 1) -#define q42(x,k) q (1, q (0, q (0, q (0, x) ^ extract_byte (k[3], 2)) ^ extract_byte (k[2], 2)) ^ extract_byte (k[1], 2)) ^ extract_byte (k[0], 2) -#define q43(x,k) q (1, q (1, q (0, q (1, x) ^ extract_byte (k[3], 3)) ^ extract_byte (k[2], 3)) ^ extract_byte (k[1], 3)) ^ extract_byte (k[0], 3) - -#define g1_fun(x) \ - (mds (0, q40 (extract_byte (x, 3), sk)) ^ \ - mds (1, q41 (extract_byte (x, 0), sk)) ^ \ - mds (2, q42 (extract_byte (x, 1), sk)) ^ \ - mds (3, q43 (extract_byte (x, 2), sk))) - -#define g0_fun(x) \ - (mds (0, q40 (extract_byte (x, 0), sk)) ^ \ - mds (1, q41 (extract_byte (x, 1), sk)) ^ \ - mds (2, q42 (extract_byte (x, 2), sk)) ^ \ - mds (3, q43 (extract_byte (x, 3), sk))) - -u32 mds_rem (u32 p0, u32 p1) -{ - #define G_MOD 0x14d - - for (int i = 0; i < 8; i++) - { - u32 t = p1 >> 24; - - p1 = (p1 << 8) | (p0 >> 24); - - p0 <<= 8; - - u32 u = (t << 1); - - if (t & 0x80) u ^= G_MOD; - - p1 ^= t ^ (u << 16); - - u ^= (t >> 1); - - if (t & 0x01) u ^= G_MOD >> 1; - - p1 ^= (u << 24) | (u << 8); - } - - return p1; -} - -void twofish256_set_key (u32 *sk, u32 *lk, const u32 *ukey) -{ - u32 me_key[4]; - - me_key[0] = ukey[0]; - me_key[1] = ukey[2]; - me_key[2] = ukey[4]; - me_key[3] = ukey[6]; - - u32 mo_key[4]; - - mo_key[0] = ukey[1]; - mo_key[1] = ukey[3]; - mo_key[2] = ukey[5]; - mo_key[3] = ukey[7]; - - sk[3] = mds_rem (me_key[0], mo_key[0]); - sk[2] = mds_rem (me_key[1], mo_key[1]); - sk[1] = mds_rem (me_key[2], mo_key[2]); - sk[0] = mds_rem (me_key[3], mo_key[3]); - - for (int i = 0; i < 40; i += 2) - { - u32 a = 0x01010101 * i; - u32 b = 0x01010101 + a; - - a = h_fun (sk, lk, a, me_key); - b = h_fun (sk, lk, b, mo_key); - - b = rotl32 (b, 8); - - lk[i + 0] = a + b; - lk[i + 1] = rotl32 (a + 2 * b, 9); - } -} - -#define f_rnd(i) \ -{ \ - u32 t0 = g0_fun (data[0]); \ - u32 t1 = g1_fun (data[1]); \ - data[2] = rotr32 (data[2] ^ (t0 + t1 + lk[4 * (i) + 8]), 1); \ - data[3] = rotl32 (data[3], 1) ^ (t0 + 2 * t1 + lk[4 * (i) + 9]); \ - u32 t2 = g0_fun (data[2]); \ - u32 t3 = g1_fun (data[3]); \ - data[0] = rotr32 (data[0] ^ (t2 + t3 + lk[4 * (i) + 10]), 1); \ - data[1] = rotl32 (data[1], 1) ^ (t2 + 2 * t3 + lk[4 * (i) + 11]); \ -} - -void twofish256_encrypt (const u32 *sk, const u32 *lk, const u32 *in, u32 *out) -{ - u32 data[4]; - - data[0] = in[0] ^ lk[0]; - data[1] = in[1] ^ lk[1]; - data[2] = in[2] ^ lk[2]; - data[3] = in[3] ^ lk[3]; - - f_rnd (0); - f_rnd (1); - f_rnd (2); - f_rnd (3); - f_rnd (4); - f_rnd (5); - f_rnd (6); - f_rnd (7); - - out[0] = data[2] ^ lk[4]; - out[1] = data[3] ^ lk[5]; - out[2] = data[0] ^ lk[6]; - out[3] = data[1] ^ lk[7]; -} - -#define i_rnd(i) \ -{ \ - u32 t0 = g0_fun (data[0]); \ - u32 t1 = g1_fun (data[1]); \ - data[2] = rotl32 (data[2], 1) ^ (t0 + t1 + lk[4 * (i) + 10]); \ - data[3] = rotr32 (data[3] ^ (t0 + 2 * t1 + lk[4 * (i) + 11]), 1); \ - u32 t2 = g0_fun (data[2]); \ - u32 t3 = g1_fun (data[3]); \ - data[0] = rotl32 (data[0], 1) ^ (t2 + t3 + lk[4 * (i) + 8]); \ - data[1] = rotr32 (data[1] ^ (t2 + 2 * t3 + lk[4 * (i) + 9]), 1); \ -} - -void twofish256_decrypt (const u32 *sk, const u32 *lk, const u32 *in, u32 *out) -{ - u32 data[4]; - - data[0] = in[0] ^ lk[4]; - data[1] = in[1] ^ lk[5]; - data[2] = in[2] ^ lk[6]; - data[3] = in[3] ^ lk[7]; - - i_rnd (7); - i_rnd (6); - i_rnd (5); - i_rnd (4); - i_rnd (3); - i_rnd (2); - i_rnd (1); - i_rnd (0); - - out[0] = data[2] ^ lk[0]; - out[1] = data[3] ^ lk[1]; - out[2] = data[0] ^ lk[2]; - out[3] = data[1] ^ lk[3]; -} - -void twofish256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out) -{ - u32 T[4] = { 0 }; - u32 Z[4] = { 0 }; - - out[0] = in[0]; - out[1] = in[1]; - out[2] = in[2]; - out[3] = in[3]; - - u32 sk[4]; u32 lk[40]; - - twofish256_set_key (sk, lk, ukey2); - twofish256_encrypt (sk, lk, Z, T); - - out[0] ^= T[0]; - out[1] ^= T[1]; - out[2] ^= T[2]; - out[3] ^= T[3]; - - twofish256_set_key (sk, lk, ukey1); - twofish256_decrypt (sk, lk, out, out); - - out[0] ^= T[0]; - out[1] ^= T[1]; - out[2] ^= T[2]; - out[3] ^= T[3]; -} diff --git a/OpenCL/m00000_a0.cl b/OpenCL/m00000_a0.cl index 08e5923..cdc550a 100644 --- a/OpenCL/m00000_a0.cl +++ b/OpenCL/m00000_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00000_a1.cl b/OpenCL/m00000_a1.cl index c89d6f2..a5df408 100644 --- a/OpenCL/m00000_a1.cl +++ b/OpenCL/m00000_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00000_a3.cl b/OpenCL/m00000_a3.cl index 09b4665..e4d86bd 100644 --- a/OpenCL/m00000_a3.cl +++ b/OpenCL/m00000_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define MD5_STEP_REV(f,a,b,c,d,x,t,s) \ { \ diff --git a/OpenCL/m00010_a0.cl b/OpenCL/m00010_a0.cl index d2beb72..416dc30 100644 --- a/OpenCL/m00010_a0.cl +++ b/OpenCL/m00010_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00010_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00010_a1.cl b/OpenCL/m00010_a1.cl index a1424c9..e3ffd36 100644 --- a/OpenCL/m00010_a1.cl +++ b/OpenCL/m00010_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00010_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00010_a3.cl b/OpenCL/m00010_a3.cl index 38ef98f..d923bd3 100644 --- a/OpenCL/m00010_a3.cl +++ b/OpenCL/m00010_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define MD5_STEP_REV(f,a,b,c,d,x,t,s) \ { \ diff --git a/OpenCL/m00020_a0.cl b/OpenCL/m00020_a0.cl index 99fc13a..db3d526 100644 --- a/OpenCL/m00020_a0.cl +++ b/OpenCL/m00020_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00020_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00020_a1.cl b/OpenCL/m00020_a1.cl index 3e5d995..8318aa2 100644 --- a/OpenCL/m00020_a1.cl +++ b/OpenCL/m00020_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00020_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00020_a3.cl b/OpenCL/m00020_a3.cl index 13d0651..d4098f9 100644 --- a/OpenCL/m00020_a3.cl +++ b/OpenCL/m00020_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m00020m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m00030_a0.cl b/OpenCL/m00030_a0.cl index 9309e30..cd86c4c 100644 --- a/OpenCL/m00030_a0.cl +++ b/OpenCL/m00030_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" // no unicode yet diff --git a/OpenCL/m00030_a1.cl b/OpenCL/m00030_a1.cl index 329c8c1..3ae93f4 100644 --- a/OpenCL/m00030_a1.cl +++ b/OpenCL/m00030_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" // no unicode yet diff --git a/OpenCL/m00030_a3.cl b/OpenCL/m00030_a3.cl index 05001ed..4b26384 100644 --- a/OpenCL/m00030_a3.cl +++ b/OpenCL/m00030_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define MD5_STEP_REV(f,a,b,c,d,x,t,s) \ { \ diff --git a/OpenCL/m00040_a0.cl b/OpenCL/m00040_a0.cl index f888f52..9521b8d 100644 --- a/OpenCL/m00040_a0.cl +++ b/OpenCL/m00040_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00040_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00040_a1.cl b/OpenCL/m00040_a1.cl index 2996c90..49c6046 100644 --- a/OpenCL/m00040_a1.cl +++ b/OpenCL/m00040_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" // no unicode yet diff --git a/OpenCL/m00040_a3.cl b/OpenCL/m00040_a3.cl index ee4ef2e..abb8c30 100644 --- a/OpenCL/m00040_a3.cl +++ b/OpenCL/m00040_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m00040m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m00050_a0.cl b/OpenCL/m00050_a0.cl index eb34668..9fc1533 100644 --- a/OpenCL/m00050_a0.cl +++ b/OpenCL/m00050_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m00050_a1.cl b/OpenCL/m00050_a1.cl index ec578f1..b269ad0 100644 --- a/OpenCL/m00050_a1.cl +++ b/OpenCL/m00050_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m00050_a3.cl b/OpenCL/m00050_a3.cl index 1e93f46..f08226e 100644 --- a/OpenCL/m00050_a3.cl +++ b/OpenCL/m00050_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m00060_a0.cl b/OpenCL/m00060_a0.cl index 687738c..e2eeddb 100644 --- a/OpenCL/m00060_a0.cl +++ b/OpenCL/m00060_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m00060_a1.cl b/OpenCL/m00060_a1.cl index 85604b8..635b081 100644 --- a/OpenCL/m00060_a1.cl +++ b/OpenCL/m00060_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m00060_a3.cl b/OpenCL/m00060_a3.cl index 2863aef..fc636d2 100644 --- a/OpenCL/m00060_a3.cl +++ b/OpenCL/m00060_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m00100_a0.cl b/OpenCL/m00100_a0.cl index 4faa35e..7cc5b2b 100644 --- a/OpenCL/m00100_a0.cl +++ b/OpenCL/m00100_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00100_a1.cl b/OpenCL/m00100_a1.cl index 51be262..39c2177 100644 --- a/OpenCL/m00100_a1.cl +++ b/OpenCL/m00100_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00100_a3.cl b/OpenCL/m00100_a3.cl index c78f744..455b840 100644 --- a/OpenCL/m00100_a3.cl +++ b/OpenCL/m00100_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m00100m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m00110_a0.cl b/OpenCL/m00110_a0.cl index e8ce8d9..09dac89 100644 --- a/OpenCL/m00110_a0.cl +++ b/OpenCL/m00110_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00110_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00110_a1.cl b/OpenCL/m00110_a1.cl index af0af26..c92ed52 100644 --- a/OpenCL/m00110_a1.cl +++ b/OpenCL/m00110_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00110_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00110_a3.cl b/OpenCL/m00110_a3.cl index b5d3fb0..b316a68 100644 --- a/OpenCL/m00110_a3.cl +++ b/OpenCL/m00110_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m00110m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m00120_a0.cl b/OpenCL/m00120_a0.cl index 61286b5..0123ccc 100644 --- a/OpenCL/m00120_a0.cl +++ b/OpenCL/m00120_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00120_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00120_a1.cl b/OpenCL/m00120_a1.cl index 1c282b9..a31c629 100644 --- a/OpenCL/m00120_a1.cl +++ b/OpenCL/m00120_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00120_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00120_a3.cl b/OpenCL/m00120_a3.cl index 98f8c02..080c497 100644 --- a/OpenCL/m00120_a3.cl +++ b/OpenCL/m00120_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m00120m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m00130_a0.cl b/OpenCL/m00130_a0.cl index c62f198..a92db01 100644 --- a/OpenCL/m00130_a0.cl +++ b/OpenCL/m00130_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" // no unicode yet diff --git a/OpenCL/m00130_a1.cl b/OpenCL/m00130_a1.cl index d766c06..6f216c5 100644 --- a/OpenCL/m00130_a1.cl +++ b/OpenCL/m00130_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" // no unicode yet diff --git a/OpenCL/m00130_a3.cl b/OpenCL/m00130_a3.cl index a9689b0..96fe640 100644 --- a/OpenCL/m00130_a3.cl +++ b/OpenCL/m00130_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m00130m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m00140_a0.cl b/OpenCL/m00140_a0.cl index f92ccb6..066f9bf 100644 --- a/OpenCL/m00140_a0.cl +++ b/OpenCL/m00140_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" // no unicode yet diff --git a/OpenCL/m00140_a1.cl b/OpenCL/m00140_a1.cl index 50cef28..36a0f92 100644 --- a/OpenCL/m00140_a1.cl +++ b/OpenCL/m00140_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" // no unicode yet diff --git a/OpenCL/m00140_a3.cl b/OpenCL/m00140_a3.cl index 0dd5758..3439437 100644 --- a/OpenCL/m00140_a3.cl +++ b/OpenCL/m00140_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m00140m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m00150_a0.cl b/OpenCL/m00150_a0.cl index 68df4b4..022208a 100644 --- a/OpenCL/m00150_a0.cl +++ b/OpenCL/m00150_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m00150_a1.cl b/OpenCL/m00150_a1.cl index 22ecebb..9a0e150 100644 --- a/OpenCL/m00150_a1.cl +++ b/OpenCL/m00150_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m00150_a3.cl b/OpenCL/m00150_a3.cl index 1e9c5ec..35d1790 100644 --- a/OpenCL/m00150_a3.cl +++ b/OpenCL/m00150_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m00160_a0.cl b/OpenCL/m00160_a0.cl index e5d8bcf..a3c0813 100644 --- a/OpenCL/m00160_a0.cl +++ b/OpenCL/m00160_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m00160_a1.cl b/OpenCL/m00160_a1.cl index 8427e86..370a7a0 100644 --- a/OpenCL/m00160_a1.cl +++ b/OpenCL/m00160_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m00160_a3.cl b/OpenCL/m00160_a3.cl index c7afc3f..4bb85c7 100644 --- a/OpenCL/m00160_a3.cl +++ b/OpenCL/m00160_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m00190_a0.cl b/OpenCL/m00190_a0.cl index 2929228..bc88d6f 100644 --- a/OpenCL/m00190_a0.cl +++ b/OpenCL/m00190_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 4 #define DGST_R2 3 #define DGST_R3 2 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00190_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00190_a1.cl b/OpenCL/m00190_a1.cl index db57ca0..ac60807 100644 --- a/OpenCL/m00190_a1.cl +++ b/OpenCL/m00190_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 4 #define DGST_R2 3 #define DGST_R3 2 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00190_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00190_a3.cl b/OpenCL/m00190_a3.cl index 15716ed..61d8ece 100644 --- a/OpenCL/m00190_a3.cl +++ b/OpenCL/m00190_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 4 #define DGST_R2 3 #define DGST_R3 2 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m00190m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m00200_a0.cl b/OpenCL/m00200_a0.cl index 8f58b40..febb350 100644 --- a/OpenCL/m00200_a0.cl +++ b/OpenCL/m00200_a0.cl @@ -8,20 +8,20 @@ //incompatible //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00200_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00200_a1.cl b/OpenCL/m00200_a1.cl index 8ce0b86..390d182 100644 --- a/OpenCL/m00200_a1.cl +++ b/OpenCL/m00200_a1.cl @@ -8,18 +8,18 @@ //incompatible //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00200_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00200_a3.cl b/OpenCL/m00200_a3.cl index 27a10c5..7605dd8 100644 --- a/OpenCL/m00200_a3.cl +++ b/OpenCL/m00200_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define ROUND(v) \ { \ diff --git a/OpenCL/m00300_a0.cl b/OpenCL/m00300_a0.cl index c083b02..c202bbc 100644 --- a/OpenCL/m00300_a0.cl +++ b/OpenCL/m00300_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00300_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00300_a1.cl b/OpenCL/m00300_a1.cl index 2af68fe..367751e 100644 --- a/OpenCL/m00300_a1.cl +++ b/OpenCL/m00300_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00300_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00300_a3.cl b/OpenCL/m00300_a3.cl index b722c7a..0f1cfce 100644 --- a/OpenCL/m00300_a3.cl +++ b/OpenCL/m00300_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m00300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m00400.cl b/OpenCL/m00400.cl index d8edbcb..c220977 100644 --- a/OpenCL/m00400.cl +++ b/OpenCL/m00400.cl @@ -7,21 +7,21 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" void md5_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) { diff --git a/OpenCL/m00500.cl b/OpenCL/m00500.cl index 6fbc6d3..462e4ee 100644 --- a/OpenCL/m00500.cl +++ b/OpenCL/m00500.cl @@ -5,20 +5,20 @@ #define _MD5_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" #define md5crypt_magic 0x00243124u diff --git a/OpenCL/m00900_a0.cl b/OpenCL/m00900_a0.cl index 376b83e..2cb123c 100644 --- a/OpenCL/m00900_a0.cl +++ b/OpenCL/m00900_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m00900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00900_a1.cl b/OpenCL/m00900_a1.cl index d7d26f5..26b2119 100644 --- a/OpenCL/m00900_a1.cl +++ b/OpenCL/m00900_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m00900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m00900_a3.cl b/OpenCL/m00900_a3.cl index 5170c0e..2bbf306 100644 --- a/OpenCL/m00900_a3.cl +++ b/OpenCL/m00900_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define MD4_STEP_REV(f,a,b,c,d,x,t,s) \ { \ diff --git a/OpenCL/m01000_a0.cl b/OpenCL/m01000_a0.cl index 70fc825..1355f15 100644 --- a/OpenCL/m01000_a0.cl +++ b/OpenCL/m01000_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m01000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01000_a1.cl b/OpenCL/m01000_a1.cl index d5455aa..2c0fcf0 100644 --- a/OpenCL/m01000_a1.cl +++ b/OpenCL/m01000_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m01000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01000_a3.cl b/OpenCL/m01000_a3.cl index f8de4c9..02dffee 100644 --- a/OpenCL/m01000_a3.cl +++ b/OpenCL/m01000_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define MD4_STEP_REV(f,a,b,c,d,x,t,s) \ { \ diff --git a/OpenCL/m01100_a0.cl b/OpenCL/m01100_a0.cl index 1cdce45..e13d20f 100644 --- a/OpenCL/m01100_a0.cl +++ b/OpenCL/m01100_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m01100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01100_a1.cl b/OpenCL/m01100_a1.cl index ebde581..fbaba9b 100644 --- a/OpenCL/m01100_a1.cl +++ b/OpenCL/m01100_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m01100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01100_a3.cl b/OpenCL/m01100_a3.cl index b1f010a..53d62d1 100644 --- a/OpenCL/m01100_a3.cl +++ b/OpenCL/m01100_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m01100m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m01400_a0.cl b/OpenCL/m01400_a0.cl index 32f694b..3c26ad6 100644 --- a/OpenCL/m01400_a0.cl +++ b/OpenCL/m01400_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m01400_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01400_a1.cl b/OpenCL/m01400_a1.cl index cc4285a..392ebf3 100644 --- a/OpenCL/m01400_a1.cl +++ b/OpenCL/m01400_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m01400_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01400_a3.cl b/OpenCL/m01400_a3.cl index 2e6b3a4..da17856 100644 --- a/OpenCL/m01400_a3.cl +++ b/OpenCL/m01400_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m01400m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m01410_a0.cl b/OpenCL/m01410_a0.cl index 2da15b8..1eb855d 100644 --- a/OpenCL/m01410_a0.cl +++ b/OpenCL/m01410_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m01410_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01410_a1.cl b/OpenCL/m01410_a1.cl index 751be42..ef9a0ce 100644 --- a/OpenCL/m01410_a1.cl +++ b/OpenCL/m01410_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m01410_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01410_a3.cl b/OpenCL/m01410_a3.cl index 13e9027..bfbc8ad 100644 --- a/OpenCL/m01410_a3.cl +++ b/OpenCL/m01410_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m01410m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m01420_a0.cl b/OpenCL/m01420_a0.cl index 5676003..13406c7 100644 --- a/OpenCL/m01420_a0.cl +++ b/OpenCL/m01420_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m01420_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01420_a1.cl b/OpenCL/m01420_a1.cl index d0a0ace..0bfed86 100644 --- a/OpenCL/m01420_a1.cl +++ b/OpenCL/m01420_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m01420_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01420_a3.cl b/OpenCL/m01420_a3.cl index a95d0c2..4f55994 100644 --- a/OpenCL/m01420_a3.cl +++ b/OpenCL/m01420_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m01420m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m01430_a0.cl b/OpenCL/m01430_a0.cl index 905beed..faa387e 100644 --- a/OpenCL/m01430_a0.cl +++ b/OpenCL/m01430_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m01430_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01430_a1.cl b/OpenCL/m01430_a1.cl index 897878f..d74da25 100644 --- a/OpenCL/m01430_a1.cl +++ b/OpenCL/m01430_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m01430_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01430_a3.cl b/OpenCL/m01430_a3.cl index a2cd4a5..cca9aa6 100644 --- a/OpenCL/m01430_a3.cl +++ b/OpenCL/m01430_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m01430m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m01440_a0.cl b/OpenCL/m01440_a0.cl index 7e38628..30b7c8a 100644 --- a/OpenCL/m01440_a0.cl +++ b/OpenCL/m01440_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m01440_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01440_a1.cl b/OpenCL/m01440_a1.cl index 8cf5364..40944ba 100644 --- a/OpenCL/m01440_a1.cl +++ b/OpenCL/m01440_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m01440_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m01440_a3.cl b/OpenCL/m01440_a3.cl index 69363a2..3e945af 100644 --- a/OpenCL/m01440_a3.cl +++ b/OpenCL/m01440_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m01440m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m01450_a0.cl b/OpenCL/m01450_a0.cl index 7e12d30..4b5a534 100644 --- a/OpenCL/m01450_a0.cl +++ b/OpenCL/m01450_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m01450_a1.cl b/OpenCL/m01450_a1.cl index ab683cf..207b8e0 100644 --- a/OpenCL/m01450_a1.cl +++ b/OpenCL/m01450_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m01450_a3.cl b/OpenCL/m01450_a3.cl index 6756571..2f2ac61 100644 --- a/OpenCL/m01450_a3.cl +++ b/OpenCL/m01450_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m01460_a0.cl b/OpenCL/m01460_a0.cl index 3824c2c..6f973a3 100644 --- a/OpenCL/m01460_a0.cl +++ b/OpenCL/m01460_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m01460_a1.cl b/OpenCL/m01460_a1.cl index 6081ef6..042753e 100644 --- a/OpenCL/m01460_a1.cl +++ b/OpenCL/m01460_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m01460_a3.cl b/OpenCL/m01460_a3.cl index 64c0df5..2ddda1b 100644 --- a/OpenCL/m01460_a3.cl +++ b/OpenCL/m01460_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m01500_a0.cl b/OpenCL/m01500_a0.cl index 0007970..041b35b 100644 --- a/OpenCL/m01500_a0.cl +++ b/OpenCL/m01500_a0.cl @@ -10,20 +10,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m01500_a1.cl b/OpenCL/m01500_a1.cl index 135cf96..7cdc7fb 100644 --- a/OpenCL/m01500_a1.cl +++ b/OpenCL/m01500_a1.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m01500_a3.cl b/OpenCL/m01500_a3.cl index 2585829..2d2e94e 100644 --- a/OpenCL/m01500_a3.cl +++ b/OpenCL/m01500_a3.cl @@ -7,20 +7,20 @@ #define _DES_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4_bs.c" -#define COMPARE_M "OpenCL/check_multi_comp4_bs.c" +#define COMPARE_S "inc_comp_single_bs.cl" +#define COMPARE_M "inc_comp_multi_bs.cl" #define myselx(a,b,c) ((c) ? (b) : (a)) diff --git a/OpenCL/m01600.cl b/OpenCL/m01600.cl index 8389cee..1b33d2a 100644 --- a/OpenCL/m01600.cl +++ b/OpenCL/m01600.cl @@ -5,20 +5,20 @@ #define _MD5_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" #define md5apr1_magic0 0x72706124u #define md5apr1_magic1 0x00002431u diff --git a/OpenCL/m01700_a0.cl b/OpenCL/m01700_a0.cl index 5324cb0..c37ed85 100644 --- a/OpenCL/m01700_a0.cl +++ b/OpenCL/m01700_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01700_a1.cl b/OpenCL/m01700_a1.cl index 57adc4f..94486f4 100644 --- a/OpenCL/m01700_a1.cl +++ b/OpenCL/m01700_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01700_a3.cl b/OpenCL/m01700_a3.cl index c7df44e..55f94f6 100644 --- a/OpenCL/m01700_a3.cl +++ b/OpenCL/m01700_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01710_a0.cl b/OpenCL/m01710_a0.cl index e2172f0..7bc0941 100644 --- a/OpenCL/m01710_a0.cl +++ b/OpenCL/m01710_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01710_a1.cl b/OpenCL/m01710_a1.cl index 794c00f..6ea7861 100644 --- a/OpenCL/m01710_a1.cl +++ b/OpenCL/m01710_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01710_a3.cl b/OpenCL/m01710_a3.cl index 794b9b3..44309b7 100644 --- a/OpenCL/m01710_a3.cl +++ b/OpenCL/m01710_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01720_a0.cl b/OpenCL/m01720_a0.cl index 52eb80c..a065011 100644 --- a/OpenCL/m01720_a0.cl +++ b/OpenCL/m01720_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01720_a1.cl b/OpenCL/m01720_a1.cl index 0ed7984..9e721e4 100644 --- a/OpenCL/m01720_a1.cl +++ b/OpenCL/m01720_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01720_a3.cl b/OpenCL/m01720_a3.cl index 12c1552..b2f5b9d 100644 --- a/OpenCL/m01720_a3.cl +++ b/OpenCL/m01720_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01730_a0.cl b/OpenCL/m01730_a0.cl index 2dd7473..7abf875 100644 --- a/OpenCL/m01730_a0.cl +++ b/OpenCL/m01730_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01730_a1.cl b/OpenCL/m01730_a1.cl index e2a0557..d6c8fe5 100644 --- a/OpenCL/m01730_a1.cl +++ b/OpenCL/m01730_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01730_a3.cl b/OpenCL/m01730_a3.cl index 517822b..0506776 100644 --- a/OpenCL/m01730_a3.cl +++ b/OpenCL/m01730_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01740_a0.cl b/OpenCL/m01740_a0.cl index d6574ce..98d49fe 100644 --- a/OpenCL/m01740_a0.cl +++ b/OpenCL/m01740_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01740_a1.cl b/OpenCL/m01740_a1.cl index c859c41..cd3c8af 100644 --- a/OpenCL/m01740_a1.cl +++ b/OpenCL/m01740_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01740_a3.cl b/OpenCL/m01740_a3.cl index 97a98d3..84d0c72 100644 --- a/OpenCL/m01740_a3.cl +++ b/OpenCL/m01740_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01750_a0.cl b/OpenCL/m01750_a0.cl index 4ba5ea2..c72482b 100644 --- a/OpenCL/m01750_a0.cl +++ b/OpenCL/m01750_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01750_a1.cl b/OpenCL/m01750_a1.cl index 65112a3..b35a789 100644 --- a/OpenCL/m01750_a1.cl +++ b/OpenCL/m01750_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01750_a3.cl b/OpenCL/m01750_a3.cl index aff816f..3de7878 100644 --- a/OpenCL/m01750_a3.cl +++ b/OpenCL/m01750_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01760_a0.cl b/OpenCL/m01760_a0.cl index 8883d47..ed2c409 100644 --- a/OpenCL/m01760_a0.cl +++ b/OpenCL/m01760_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01760_a1.cl b/OpenCL/m01760_a1.cl index 1d7895f..eaa8624 100644 --- a/OpenCL/m01760_a1.cl +++ b/OpenCL/m01760_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01760_a3.cl b/OpenCL/m01760_a3.cl index 61bb6d7..5bf5380 100644 --- a/OpenCL/m01760_a3.cl +++ b/OpenCL/m01760_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 14 #define DGST_R1 15 #define DGST_R2 6 #define DGST_R3 7 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m01800.cl b/OpenCL/m01800.cl index 21b1685..4af9eb6 100644 --- a/OpenCL/m01800.cl +++ b/OpenCL/m01800.cl @@ -5,20 +5,20 @@ #define _SHA512_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" // Buggy drivers... diff --git a/OpenCL/m02100.cl b/OpenCL/m02100.cl index 2e2a8d2..89e45f8 100644 --- a/OpenCL/m02100.cl +++ b/OpenCL/m02100.cl @@ -7,21 +7,21 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" void md4_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) { diff --git a/OpenCL/m02400_a0.cl b/OpenCL/m02400_a0.cl index 0d8dc34..592c3e9 100644 --- a/OpenCL/m02400_a0.cl +++ b/OpenCL/m02400_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m02400_m04 (__global pw_t *pws, __global kernel_rule_t * rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m02400_a1.cl b/OpenCL/m02400_a1.cl index f434286..452ddb7 100644 --- a/OpenCL/m02400_a1.cl +++ b/OpenCL/m02400_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m02400_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m02400_a3.cl b/OpenCL/m02400_a3.cl index 243bd9c..937bd30 100644 --- a/OpenCL/m02400_a3.cl +++ b/OpenCL/m02400_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m02400m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m02410_a0.cl b/OpenCL/m02410_a0.cl index bb1ce35..c9d37a8 100644 --- a/OpenCL/m02410_a0.cl +++ b/OpenCL/m02410_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m02410_m04 (__global pw_t *pws, __global kernel_rule_t * rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m02410_a1.cl b/OpenCL/m02410_a1.cl index a6b0a59..4e3974f 100644 --- a/OpenCL/m02410_a1.cl +++ b/OpenCL/m02410_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m02410_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m02410_a3.cl b/OpenCL/m02410_a3.cl index d1e0679..4aedaef 100644 --- a/OpenCL/m02410_a3.cl +++ b/OpenCL/m02410_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m02410m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m02500.cl b/OpenCL/m02500.cl index 78999f6..9f90c54 100644 --- a/OpenCL/m02500.cl +++ b/OpenCL/m02500.cl @@ -7,21 +7,21 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" void md5_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) { diff --git a/OpenCL/m02610_a0.cl b/OpenCL/m02610_a0.cl index 57c7afc..ab4eaf1 100644 --- a/OpenCL/m02610_a0.cl +++ b/OpenCL/m02610_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m02610_a1.cl b/OpenCL/m02610_a1.cl index 54e90e9..eac506d 100644 --- a/OpenCL/m02610_a1.cl +++ b/OpenCL/m02610_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m02610_a3.cl b/OpenCL/m02610_a3.cl index a7a3c81..677d69c 100644 --- a/OpenCL/m02610_a3.cl +++ b/OpenCL/m02610_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m02710_a0.cl b/OpenCL/m02710_a0.cl index c3ca3e3..3fbf245 100644 --- a/OpenCL/m02710_a0.cl +++ b/OpenCL/m02710_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m02710_a1.cl b/OpenCL/m02710_a1.cl index 8748ac4..37a617b 100644 --- a/OpenCL/m02710_a1.cl +++ b/OpenCL/m02710_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m02710_a3.cl b/OpenCL/m02710_a3.cl index 23b3a25..dd73e7c 100644 --- a/OpenCL/m02710_a3.cl +++ b/OpenCL/m02710_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m02810_a0.cl b/OpenCL/m02810_a0.cl index 5cdd1e6..c9458fe 100644 --- a/OpenCL/m02810_a0.cl +++ b/OpenCL/m02810_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m02810_a1.cl b/OpenCL/m02810_a1.cl index 768f535..19ea018 100644 --- a/OpenCL/m02810_a1.cl +++ b/OpenCL/m02810_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m02810_a3.cl b/OpenCL/m02810_a3.cl index 6ec3014..cdcce5e 100644 --- a/OpenCL/m02810_a3.cl +++ b/OpenCL/m02810_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m03000_a0.cl b/OpenCL/m03000_a0.cl index dc3653d..10ad91a 100644 --- a/OpenCL/m03000_a0.cl +++ b/OpenCL/m03000_a0.cl @@ -10,20 +10,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m03000_a1.cl b/OpenCL/m03000_a1.cl index 22b6de6..d500e0a 100644 --- a/OpenCL/m03000_a1.cl +++ b/OpenCL/m03000_a1.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m03000_a3.cl b/OpenCL/m03000_a3.cl index f5d3380..cf501f1 100644 --- a/OpenCL/m03000_a3.cl +++ b/OpenCL/m03000_a3.cl @@ -6,20 +6,20 @@ #define _DES_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4_bs.c" -#define COMPARE_M "OpenCL/check_multi_comp4_bs.c" +#define COMPARE_S "inc_comp_single_bs.cl" +#define COMPARE_M "inc_comp_multi_bs.cl" #ifdef IS_NV #define KXX_DECL diff --git a/OpenCL/m03100_a0.cl b/OpenCL/m03100_a0.cl index 5f98fda..df52606 100644 --- a/OpenCL/m03100_a0.cl +++ b/OpenCL/m03100_a0.cl @@ -11,20 +11,20 @@ //incompatible //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m03100_a1.cl b/OpenCL/m03100_a1.cl index b16db6c..18d7741 100644 --- a/OpenCL/m03100_a1.cl +++ b/OpenCL/m03100_a1.cl @@ -10,18 +10,18 @@ //incompatible //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m03100_a3.cl b/OpenCL/m03100_a3.cl index e1729e3..5c25677 100644 --- a/OpenCL/m03100_a3.cl +++ b/OpenCL/m03100_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m03200.cl b/OpenCL/m03200.cl index a3789cd..a278122 100644 --- a/OpenCL/m03200.cl +++ b/OpenCL/m03200.cl @@ -5,20 +5,20 @@ #define _BCRYPT_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" // http://www.schneier.com/code/constants.txt diff --git a/OpenCL/m03710_a0.cl b/OpenCL/m03710_a0.cl index 4ee6320..74aa5e8 100644 --- a/OpenCL/m03710_a0.cl +++ b/OpenCL/m03710_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m03710_a1.cl b/OpenCL/m03710_a1.cl index f3f777a..c164fdf 100644 --- a/OpenCL/m03710_a1.cl +++ b/OpenCL/m03710_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m03710_a3.cl b/OpenCL/m03710_a3.cl index 7d3a1ab..9e5eacd 100644 --- a/OpenCL/m03710_a3.cl +++ b/OpenCL/m03710_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m03800_a0.cl b/OpenCL/m03800_a0.cl index f4b311e..ee926d8 100644 --- a/OpenCL/m03800_a0.cl +++ b/OpenCL/m03800_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m03800_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m03800_a1.cl b/OpenCL/m03800_a1.cl index e860d07..f727a02 100644 --- a/OpenCL/m03800_a1.cl +++ b/OpenCL/m03800_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m03800_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m03800_a3.cl b/OpenCL/m03800_a3.cl index 9b31135..79af82b 100644 --- a/OpenCL/m03800_a3.cl +++ b/OpenCL/m03800_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m03800m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m04310_a0.cl b/OpenCL/m04310_a0.cl index 0cfd0cd..038f50d 100644 --- a/OpenCL/m04310_a0.cl +++ b/OpenCL/m04310_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04310_a1.cl b/OpenCL/m04310_a1.cl index f4cb7c1..356fd0e 100644 --- a/OpenCL/m04310_a1.cl +++ b/OpenCL/m04310_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04310_a3.cl b/OpenCL/m04310_a3.cl index 5b50455..25c2a75 100644 --- a/OpenCL/m04310_a3.cl +++ b/OpenCL/m04310_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04400_a0.cl b/OpenCL/m04400_a0.cl index f2b7f76..e6f65a4 100644 --- a/OpenCL/m04400_a0.cl +++ b/OpenCL/m04400_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04400_a1.cl b/OpenCL/m04400_a1.cl index 435e041..64a727b 100644 --- a/OpenCL/m04400_a1.cl +++ b/OpenCL/m04400_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04400_a3.cl b/OpenCL/m04400_a3.cl index 76fb33b..3c587c6 100644 --- a/OpenCL/m04400_a3.cl +++ b/OpenCL/m04400_a3.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04500_a0.cl b/OpenCL/m04500_a0.cl index 56076a4..7e71655 100644 --- a/OpenCL/m04500_a0.cl +++ b/OpenCL/m04500_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04500_a1.cl b/OpenCL/m04500_a1.cl index 4fe75a1..d58956b 100644 --- a/OpenCL/m04500_a1.cl +++ b/OpenCL/m04500_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04500_a3.cl b/OpenCL/m04500_a3.cl index 2d5ff62..f2c59f2 100644 --- a/OpenCL/m04500_a3.cl +++ b/OpenCL/m04500_a3.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04700_a0.cl b/OpenCL/m04700_a0.cl index bc73392..3d06321 100644 --- a/OpenCL/m04700_a0.cl +++ b/OpenCL/m04700_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04700_a1.cl b/OpenCL/m04700_a1.cl index b71ef75..3f572d3 100644 --- a/OpenCL/m04700_a1.cl +++ b/OpenCL/m04700_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04700_a3.cl b/OpenCL/m04700_a3.cl index a1e13ee..a5f28cb 100644 --- a/OpenCL/m04700_a3.cl +++ b/OpenCL/m04700_a3.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04800_a0.cl b/OpenCL/m04800_a0.cl index 5df1807..72183d0 100644 --- a/OpenCL/m04800_a0.cl +++ b/OpenCL/m04800_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m04800_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m04800_a1.cl b/OpenCL/m04800_a1.cl index 7af41b7..4258491 100644 --- a/OpenCL/m04800_a1.cl +++ b/OpenCL/m04800_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m04800_a3.cl b/OpenCL/m04800_a3.cl index eea003d..52445bc 100644 --- a/OpenCL/m04800_a3.cl +++ b/OpenCL/m04800_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m04800m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m04900_a0.cl b/OpenCL/m04900_a0.cl index 15d8ec2..6ad2453 100644 --- a/OpenCL/m04900_a0.cl +++ b/OpenCL/m04900_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m04900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m04900_a1.cl b/OpenCL/m04900_a1.cl index 31b9ae4..79be593 100644 --- a/OpenCL/m04900_a1.cl +++ b/OpenCL/m04900_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m04900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m04900_a3.cl b/OpenCL/m04900_a3.cl index 9cf2ba0..b7b75b8 100644 --- a/OpenCL/m04900_a3.cl +++ b/OpenCL/m04900_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m04900m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m05000_a0.cl b/OpenCL/m05000_a0.cl index 7bada0e..8d53020 100644 --- a/OpenCL/m05000_a0.cl +++ b/OpenCL/m05000_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 2 #define DGST_R1 3 #define DGST_R2 4 #define DGST_R3 5 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u64 keccakf_rndc[24] = { diff --git a/OpenCL/m05000_a1.cl b/OpenCL/m05000_a1.cl index 7d22c13..b6ef979 100644 --- a/OpenCL/m05000_a1.cl +++ b/OpenCL/m05000_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 2 #define DGST_R1 3 #define DGST_R2 4 #define DGST_R3 5 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 keccakf_rndc[24] = { diff --git a/OpenCL/m05000_a3.cl b/OpenCL/m05000_a3.cl index e057e16..30c4ea1 100644 --- a/OpenCL/m05000_a3.cl +++ b/OpenCL/m05000_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 2 #define DGST_R1 3 #define DGST_R2 4 #define DGST_R3 5 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 keccakf_rndc[24] = { diff --git a/OpenCL/m05100_a0.cl b/OpenCL/m05100_a0.cl index 5f6593b..e3625a0 100644 --- a/OpenCL/m05100_a0.cl +++ b/OpenCL/m05100_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m05100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m05100_a1.cl b/OpenCL/m05100_a1.cl index 795f883..7b0f470 100644 --- a/OpenCL/m05100_a1.cl +++ b/OpenCL/m05100_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m05100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m05100_a3.cl b/OpenCL/m05100_a3.cl index 5a41ae2..7aa9226 100644 --- a/OpenCL/m05100_a3.cl +++ b/OpenCL/m05100_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m05100m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m05200.cl b/OpenCL/m05200.cl index abd2912..2fb4547 100644 --- a/OpenCL/m05200.cl +++ b/OpenCL/m05200.cl @@ -5,20 +5,20 @@ #define _SHA256_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m05300_a0.cl b/OpenCL/m05300_a0.cl index 943589d..0fc2f1a 100644 --- a/OpenCL/m05300_a0.cl +++ b/OpenCL/m05300_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m05300_a1.cl b/OpenCL/m05300_a1.cl index 9f4e6e9..7c53c3b 100644 --- a/OpenCL/m05300_a1.cl +++ b/OpenCL/m05300_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m05300_a3.cl b/OpenCL/m05300_a3.cl index ee2d43c..276c2a3 100644 --- a/OpenCL/m05300_a3.cl +++ b/OpenCL/m05300_a3.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m05400_a0.cl b/OpenCL/m05400_a0.cl index d881a58..a722c53 100644 --- a/OpenCL/m05400_a0.cl +++ b/OpenCL/m05400_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m05400_a1.cl b/OpenCL/m05400_a1.cl index 4958b39..ec2955d 100644 --- a/OpenCL/m05400_a1.cl +++ b/OpenCL/m05400_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m05400_a3.cl b/OpenCL/m05400_a3.cl index d4d91fb..f76af43 100644 --- a/OpenCL/m05400_a3.cl +++ b/OpenCL/m05400_a3.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m05500_a0.cl b/OpenCL/m05500_a0.cl index 82ad293..90b7ad6 100644 --- a/OpenCL/m05500_a0.cl +++ b/OpenCL/m05500_a0.cl @@ -10,20 +10,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m05500_a1.cl b/OpenCL/m05500_a1.cl index be2994c..12d2d48 100644 --- a/OpenCL/m05500_a1.cl +++ b/OpenCL/m05500_a1.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m05500_a3.cl b/OpenCL/m05500_a3.cl index a9fc359..d77bb3b 100644 --- a/OpenCL/m05500_a3.cl +++ b/OpenCL/m05500_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m05600_a0.cl b/OpenCL/m05600_a0.cl index 74920b7..678cca8 100644 --- a/OpenCL/m05600_a0.cl +++ b/OpenCL/m05600_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void md4_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m05600_a1.cl b/OpenCL/m05600_a1.cl index e0d566b..8ce3e69 100644 --- a/OpenCL/m05600_a1.cl +++ b/OpenCL/m05600_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md4_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m05600_a3.cl b/OpenCL/m05600_a3.cl index 799dce0..90b28a5 100644 --- a/OpenCL/m05600_a3.cl +++ b/OpenCL/m05600_a3.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md4_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m05800.cl b/OpenCL/m05800.cl index 1c363b7..fda4748 100644 --- a/OpenCL/m05800.cl +++ b/OpenCL/m05800.cl @@ -5,20 +5,20 @@ #define _SHA1_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" typedef struct { diff --git a/OpenCL/m06000_a0.cl b/OpenCL/m06000_a0.cl index 5f630f8..a9b4310 100644 --- a/OpenCL/m06000_a0.cl +++ b/OpenCL/m06000_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void ripemd160_transform (const u32x w[16], u32x dgst[5]) { diff --git a/OpenCL/m06000_a1.cl b/OpenCL/m06000_a1.cl index 29e546c..77d832e 100644 --- a/OpenCL/m06000_a1.cl +++ b/OpenCL/m06000_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void ripemd160_transform (const u32x w[16], u32x dgst[5]) { diff --git a/OpenCL/m06000_a3.cl b/OpenCL/m06000_a3.cl index ac3eab8..c9222cd 100644 --- a/OpenCL/m06000_a3.cl +++ b/OpenCL/m06000_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void ripemd160_transform (const u32x w[16], u32x dgst[5]) { diff --git a/OpenCL/m06100_a0.cl b/OpenCL/m06100_a0.cl index 3b0d23c..085f790 100644 --- a/OpenCL/m06100_a0.cl +++ b/OpenCL/m06100_a0.cl @@ -10,20 +10,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define R 10 diff --git a/OpenCL/m06100_a1.cl b/OpenCL/m06100_a1.cl index 35005ae..2280f21 100644 --- a/OpenCL/m06100_a1.cl +++ b/OpenCL/m06100_a1.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define R 10 diff --git a/OpenCL/m06100_a3.cl b/OpenCL/m06100_a3.cl index d613d2e..c26d3fb 100644 --- a/OpenCL/m06100_a3.cl +++ b/OpenCL/m06100_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define R 10 diff --git a/OpenCL/m06211.cl b/OpenCL/m06211.cl index da07d43..f3895da 100644 --- a/OpenCL/m06211.cl +++ b/OpenCL/m06211.cl @@ -5,21 +5,21 @@ #define _RIPEMD160_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" void ripemd160_transform (const u32 w[16], u32 dgst[5]) { diff --git a/OpenCL/m06212.cl b/OpenCL/m06212.cl index 7262ac3..d06cfd8 100644 --- a/OpenCL/m06212.cl +++ b/OpenCL/m06212.cl @@ -5,21 +5,21 @@ #define _RIPEMD160_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" void ripemd160_transform (const u32 w[16], u32 dgst[5]) { diff --git a/OpenCL/m06213.cl b/OpenCL/m06213.cl index 0b49be0..61a85c1 100644 --- a/OpenCL/m06213.cl +++ b/OpenCL/m06213.cl @@ -5,21 +5,21 @@ #define _RIPEMD160_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" void ripemd160_transform (const u32 w[16], u32 dgst[5]) { diff --git a/OpenCL/m06221.cl b/OpenCL/m06221.cl index 5959263..be8041f 100644 --- a/OpenCL/m06221.cl +++ b/OpenCL/m06221.cl @@ -5,21 +5,21 @@ #define _SHA512_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m06222.cl b/OpenCL/m06222.cl index e416c66..1c09496 100644 --- a/OpenCL/m06222.cl +++ b/OpenCL/m06222.cl @@ -5,21 +5,21 @@ #define _SHA512_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m06223.cl b/OpenCL/m06223.cl index c2007b0..3e7aa8d 100644 --- a/OpenCL/m06223.cl +++ b/OpenCL/m06223.cl @@ -5,21 +5,21 @@ #define _SHA512_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m06231.cl b/OpenCL/m06231.cl index 90f248c..2ead1ea 100644 --- a/OpenCL/m06231.cl +++ b/OpenCL/m06231.cl @@ -7,21 +7,21 @@ #define _WHIRLPOOL_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" #define R 10 diff --git a/OpenCL/m06232.cl b/OpenCL/m06232.cl index 942de68..b10280e 100644 --- a/OpenCL/m06232.cl +++ b/OpenCL/m06232.cl @@ -7,21 +7,21 @@ #define _WHIRLPOOL_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" #define R 10 diff --git a/OpenCL/m06233.cl b/OpenCL/m06233.cl index acb6719..0235544 100644 --- a/OpenCL/m06233.cl +++ b/OpenCL/m06233.cl @@ -7,21 +7,21 @@ #define _WHIRLPOOL_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" #define R 10 diff --git a/OpenCL/m06300.cl b/OpenCL/m06300.cl index f4dcf90..cfdba3b 100644 --- a/OpenCL/m06300.cl +++ b/OpenCL/m06300.cl @@ -5,20 +5,20 @@ #define _MD5_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" void md5_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) { diff --git a/OpenCL/m06400.cl b/OpenCL/m06400.cl index 99e0dd8..8c632a5 100644 --- a/OpenCL/m06400.cl +++ b/OpenCL/m06400.cl @@ -5,20 +5,20 @@ #define _SHA256_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m06500.cl b/OpenCL/m06500.cl index b5d77e1..5f23af1 100644 --- a/OpenCL/m06500.cl +++ b/OpenCL/m06500.cl @@ -5,20 +5,20 @@ #define _SHA512_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m06600.cl b/OpenCL/m06600.cl index 6ee0523..ca2f14e 100644 --- a/OpenCL/m06600.cl +++ b/OpenCL/m06600.cl @@ -7,20 +7,20 @@ #define _SHA1_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m06700.cl b/OpenCL/m06700.cl index 3f8aceb..f8b5ff8 100644 --- a/OpenCL/m06700.cl +++ b/OpenCL/m06700.cl @@ -5,20 +5,20 @@ #define _SHA1_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" void sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5]) { diff --git a/OpenCL/m06800.cl b/OpenCL/m06800.cl index d2e6915..6744d6a 100644 --- a/OpenCL/m06800.cl +++ b/OpenCL/m06800.cl @@ -7,20 +7,20 @@ #define _SHA256_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m06900_a0.cl b/OpenCL/m06900_a0.cl index 82833bf..bbefaff 100644 --- a/OpenCL/m06900_a0.cl +++ b/OpenCL/m06900_a0.cl @@ -10,20 +10,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 c_tables[4][256] = { diff --git a/OpenCL/m06900_a1.cl b/OpenCL/m06900_a1.cl index f7a0765..fedc7fa 100644 --- a/OpenCL/m06900_a1.cl +++ b/OpenCL/m06900_a1.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 c_tables[4][256] = { diff --git a/OpenCL/m06900_a3.cl b/OpenCL/m06900_a3.cl index dc7c705..66f6f82 100644 --- a/OpenCL/m06900_a3.cl +++ b/OpenCL/m06900_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 c_tables[4][256] = { diff --git a/OpenCL/m07100.cl b/OpenCL/m07100.cl index 1147122..6f64405 100644 --- a/OpenCL/m07100.cl +++ b/OpenCL/m07100.cl @@ -7,21 +7,21 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m07300_a0.cl b/OpenCL/m07300_a0.cl index 9d4dccf..27c7a4f 100644 --- a/OpenCL/m07300_a0.cl +++ b/OpenCL/m07300_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m07300_a1.cl b/OpenCL/m07300_a1.cl index 2cfa5ca..b78560c 100644 --- a/OpenCL/m07300_a1.cl +++ b/OpenCL/m07300_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m07300_a3.cl b/OpenCL/m07300_a3.cl index 944ec2e..25a0495 100644 --- a/OpenCL/m07300_a3.cl +++ b/OpenCL/m07300_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m07400.cl b/OpenCL/m07400.cl index 0e22a5e..3fc4600 100644 --- a/OpenCL/m07400.cl +++ b/OpenCL/m07400.cl @@ -5,20 +5,20 @@ #define _SHA256_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m07500_a0.cl b/OpenCL/m07500_a0.cl index 0393d8d..ea2e219 100644 --- a/OpenCL/m07500_a0.cl +++ b/OpenCL/m07500_a0.cl @@ -8,20 +8,20 @@ //shared mem too small //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m07500_a1.cl b/OpenCL/m07500_a1.cl index 6983b16..673df0a 100644 --- a/OpenCL/m07500_a1.cl +++ b/OpenCL/m07500_a1.cl @@ -8,18 +8,18 @@ //shared mem too small //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m07500_a3.cl b/OpenCL/m07500_a3.cl index 1e8eb93..dd57cce 100644 --- a/OpenCL/m07500_a3.cl +++ b/OpenCL/m07500_a3.cl @@ -10,18 +10,18 @@ //shared mem too small //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m07600_a0.cl b/OpenCL/m07600_a0.cl index 86a0012..15d1ed1 100644 --- a/OpenCL/m07600_a0.cl +++ b/OpenCL/m07600_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m07600_a1.cl b/OpenCL/m07600_a1.cl index 531d5f3..b904ed4 100644 --- a/OpenCL/m07600_a1.cl +++ b/OpenCL/m07600_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m07600_a3.cl b/OpenCL/m07600_a3.cl index 2e068e7..717d7de 100644 --- a/OpenCL/m07600_a3.cl +++ b/OpenCL/m07600_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m07700_a0.cl b/OpenCL/m07700_a0.cl index 2755132..671d02d 100644 --- a/OpenCL/m07700_a0.cl +++ b/OpenCL/m07700_a0.cl @@ -8,20 +8,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define GETCHAR(a,p) (((a)[(p) / 4] >> (((p) & 3) * 8)) & 0xff) #define PUTCHAR(a,p,c) ((a)[(p) / 4] = (((a)[(p) / 4] & ~(0xff << (((p) & 3) * 8))) | ((c) << (((p) & 3) * 8)))) diff --git a/OpenCL/m07700_a1.cl b/OpenCL/m07700_a1.cl index 8eb9054..bf98509 100644 --- a/OpenCL/m07700_a1.cl +++ b/OpenCL/m07700_a1.cl @@ -8,18 +8,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define GETCHAR(a,p) (((a)[(p) / 4] >> (((p) & 3) * 8)) & 0xff) #define PUTCHAR(a,p,c) ((a)[(p) / 4] = (((a)[(p) / 4] & ~(0xff << (((p) & 3) * 8))) | ((c) << (((p) & 3) * 8)))) diff --git a/OpenCL/m07700_a3.cl b/OpenCL/m07700_a3.cl index dac0379..293cc49 100644 --- a/OpenCL/m07700_a3.cl +++ b/OpenCL/m07700_a3.cl @@ -8,18 +8,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define GETCHAR(a,p) (((a)[(p) / 4] >> (((p) & 3) * 8)) & 0xff) #define PUTCHAR(a,p,c) ((a)[(p) / 4] = (((a)[(p) / 4] & ~(0xff << (((p) & 3) * 8))) | ((c) << (((p) & 3) * 8)))) diff --git a/OpenCL/m07800_a0.cl b/OpenCL/m07800_a0.cl index b4b98db..308b582 100644 --- a/OpenCL/m07800_a0.cl +++ b/OpenCL/m07800_a0.cl @@ -8,20 +8,20 @@ //incompatible data-dependant code //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define GETSHIFTEDINT(a,n) amd_bytealign ((a)[((n)/4)+1], (a)[((n)/4)+0], (n)) diff --git a/OpenCL/m07800_a1.cl b/OpenCL/m07800_a1.cl index feb36f1..4fd07eb 100644 --- a/OpenCL/m07800_a1.cl +++ b/OpenCL/m07800_a1.cl @@ -8,18 +8,18 @@ //incompatible data-dependant code //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define GETSHIFTEDINT(a,n) amd_bytealign ((a)[((n)/4)+1], (a)[((n)/4)+0], (n)) diff --git a/OpenCL/m07800_a3.cl b/OpenCL/m07800_a3.cl index 3b75ace..7d1ce5a 100644 --- a/OpenCL/m07800_a3.cl +++ b/OpenCL/m07800_a3.cl @@ -8,18 +8,18 @@ //incompatible data-dependant code //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define GETSHIFTEDINT(a,n) amd_bytealign ((a)[((n)/4)+1], (a)[((n)/4)+0], (n)) diff --git a/OpenCL/m07900.cl b/OpenCL/m07900.cl index 7226bec..49389d8 100644 --- a/OpenCL/m07900.cl +++ b/OpenCL/m07900.cl @@ -5,20 +5,20 @@ #define _SHA512_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m08000_a0.cl b/OpenCL/m08000_a0.cl index 33266e0..046b84e 100644 --- a/OpenCL/m08000_a0.cl +++ b/OpenCL/m08000_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m08000_a1.cl b/OpenCL/m08000_a1.cl index efbc2bb..9ac3580 100644 --- a/OpenCL/m08000_a1.cl +++ b/OpenCL/m08000_a1.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m08000_a3.cl b/OpenCL/m08000_a3.cl index b670305..01f66fb 100644 --- a/OpenCL/m08000_a3.cl +++ b/OpenCL/m08000_a3.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m08100_a0.cl b/OpenCL/m08100_a0.cl index 95b76e2..32c8a61 100644 --- a/OpenCL/m08100_a0.cl +++ b/OpenCL/m08100_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m08100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m08100_a1.cl b/OpenCL/m08100_a1.cl index 25980ec..a7c2d8d 100644 --- a/OpenCL/m08100_a1.cl +++ b/OpenCL/m08100_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m08100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m08100_a3.cl b/OpenCL/m08100_a3.cl index 4848662..58240d1 100644 --- a/OpenCL/m08100_a3.cl +++ b/OpenCL/m08100_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m08100m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m08200.cl b/OpenCL/m08200.cl index dc7b768..ae7e6de 100644 --- a/OpenCL/m08200.cl +++ b/OpenCL/m08200.cl @@ -5,20 +5,20 @@ #define _CLOUDKEY_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m08300_a0.cl b/OpenCL/m08300_a0.cl index 904f680..84dfa7f 100644 --- a/OpenCL/m08300_a0.cl +++ b/OpenCL/m08300_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m08300_a1.cl b/OpenCL/m08300_a1.cl index e36b357..b22986f 100644 --- a/OpenCL/m08300_a1.cl +++ b/OpenCL/m08300_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m08300_a3.cl b/OpenCL/m08300_a3.cl index 9de6a42..b0e73e1 100644 --- a/OpenCL/m08300_a3.cl +++ b/OpenCL/m08300_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m08400_a0.cl b/OpenCL/m08400_a0.cl index 2fa98bd..06df81f 100644 --- a/OpenCL/m08400_a0.cl +++ b/OpenCL/m08400_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m08400_a1.cl b/OpenCL/m08400_a1.cl index 5cb4be4..e408c05 100644 --- a/OpenCL/m08400_a1.cl +++ b/OpenCL/m08400_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m08400_a3.cl b/OpenCL/m08400_a3.cl index 7a089ad..ce4c926 100644 --- a/OpenCL/m08400_a3.cl +++ b/OpenCL/m08400_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m08500_a0.cl b/OpenCL/m08500_a0.cl index d21d101..6874a7e 100644 --- a/OpenCL/m08500_a0.cl +++ b/OpenCL/m08500_a0.cl @@ -10,20 +10,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m08500_a1.cl b/OpenCL/m08500_a1.cl index 20f2599..d007a1c 100644 --- a/OpenCL/m08500_a1.cl +++ b/OpenCL/m08500_a1.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m08500_a3.cl b/OpenCL/m08500_a3.cl index a97d62d..6e5058a 100644 --- a/OpenCL/m08500_a3.cl +++ b/OpenCL/m08500_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m08600_a0.cl b/OpenCL/m08600_a0.cl index c4a043c..c6e58ed 100644 --- a/OpenCL/m08600_a0.cl +++ b/OpenCL/m08600_a0.cl @@ -11,20 +11,20 @@ //incompatible //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 lotus_magic_table[256] = { diff --git a/OpenCL/m08600_a1.cl b/OpenCL/m08600_a1.cl index 58be048..517bbe1 100644 --- a/OpenCL/m08600_a1.cl +++ b/OpenCL/m08600_a1.cl @@ -10,18 +10,18 @@ //incompatible //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 lotus_magic_table[256] = { diff --git a/OpenCL/m08600_a3.cl b/OpenCL/m08600_a3.cl index 5b7d924..5fe5e44 100644 --- a/OpenCL/m08600_a3.cl +++ b/OpenCL/m08600_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 lotus_magic_table[256] = { diff --git a/OpenCL/m08700_a0.cl b/OpenCL/m08700_a0.cl index 1bcb8d6..9506789 100644 --- a/OpenCL/m08700_a0.cl +++ b/OpenCL/m08700_a0.cl @@ -11,20 +11,20 @@ //incompatible //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 lotus_magic_table[256] = { diff --git a/OpenCL/m08700_a1.cl b/OpenCL/m08700_a1.cl index ed6d197..00650f1 100644 --- a/OpenCL/m08700_a1.cl +++ b/OpenCL/m08700_a1.cl @@ -10,18 +10,18 @@ //incompatible //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 lotus_magic_table[256] = { diff --git a/OpenCL/m08700_a3.cl b/OpenCL/m08700_a3.cl index b6fd2d3..f3bd318 100644 --- a/OpenCL/m08700_a3.cl +++ b/OpenCL/m08700_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 lotus_magic_table[256] = { diff --git a/OpenCL/m08800.cl b/OpenCL/m08800.cl index 38a3033..50c5ca1 100644 --- a/OpenCL/m08800.cl +++ b/OpenCL/m08800.cl @@ -7,17 +7,17 @@ #define _ANDROIDFDE_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m08900.cl b/OpenCL/m08900.cl index 85ad68d..c2a718a 100644 --- a/OpenCL/m08900.cl +++ b/OpenCL/m08900.cl @@ -5,20 +5,20 @@ #define _SCRYPT_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m09000.cl b/OpenCL/m09000.cl index 210b620..7b1166b 100644 --- a/OpenCL/m09000.cl +++ b/OpenCL/m09000.cl @@ -5,20 +5,20 @@ #define _PSAFE2_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" // http://www.schneier.com/code/constants.txt diff --git a/OpenCL/m09100.cl b/OpenCL/m09100.cl index 4efe528..2d694d6 100644 --- a/OpenCL/m09100.cl +++ b/OpenCL/m09100.cl @@ -7,23 +7,23 @@ #define _LOTUS8_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" +#include "inc_hash_functions.cl" #undef _SHA1_ -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant char lotus64_table[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+/"; diff --git a/OpenCL/m09400.cl b/OpenCL/m09400.cl index d9249dc..d28c6b2 100644 --- a/OpenCL/m09400.cl +++ b/OpenCL/m09400.cl @@ -7,20 +7,20 @@ #define _OFFICE2007_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m09500.cl b/OpenCL/m09500.cl index 3445f43..a51d1fc 100644 --- a/OpenCL/m09500.cl +++ b/OpenCL/m09500.cl @@ -7,20 +7,20 @@ #define _OFFICE2010_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m09600.cl b/OpenCL/m09600.cl index eb28f6c..18acd46 100644 --- a/OpenCL/m09600.cl +++ b/OpenCL/m09600.cl @@ -7,20 +7,20 @@ #define _OFFICE2013_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m09700_a0.cl b/OpenCL/m09700_a0.cl index 4fa60b9..0c71129 100644 --- a/OpenCL/m09700_a0.cl +++ b/OpenCL/m09700_a0.cl @@ -8,20 +8,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09700_a1.cl b/OpenCL/m09700_a1.cl index 65162f7..1fcb15e 100644 --- a/OpenCL/m09700_a1.cl +++ b/OpenCL/m09700_a1.cl @@ -8,18 +8,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09700_a3.cl b/OpenCL/m09700_a3.cl index 45128af..d9a4cf9 100644 --- a/OpenCL/m09700_a3.cl +++ b/OpenCL/m09700_a3.cl @@ -7,18 +7,18 @@ #define _OLDOFFICE01_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09710_a0.cl b/OpenCL/m09710_a0.cl index b992e8d..040abd5 100644 --- a/OpenCL/m09710_a0.cl +++ b/OpenCL/m09710_a0.cl @@ -8,20 +8,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09710_a1.cl b/OpenCL/m09710_a1.cl index c761e44..74e119e 100644 --- a/OpenCL/m09710_a1.cl +++ b/OpenCL/m09710_a1.cl @@ -8,18 +8,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09710_a3.cl b/OpenCL/m09710_a3.cl index 61f23d1..17b4adf 100644 --- a/OpenCL/m09710_a3.cl +++ b/OpenCL/m09710_a3.cl @@ -10,18 +10,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09720_a0.cl b/OpenCL/m09720_a0.cl index 6721d00..8aeb1e2 100644 --- a/OpenCL/m09720_a0.cl +++ b/OpenCL/m09720_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m09720_a1.cl b/OpenCL/m09720_a1.cl index 091d041..b0c6bba 100644 --- a/OpenCL/m09720_a1.cl +++ b/OpenCL/m09720_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m09720_a3.cl b/OpenCL/m09720_a3.cl index 0b904da..66b5ccd 100644 --- a/OpenCL/m09720_a3.cl +++ b/OpenCL/m09720_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) { diff --git a/OpenCL/m09800_a0.cl b/OpenCL/m09800_a0.cl index 31f6355..2267b1a 100644 --- a/OpenCL/m09800_a0.cl +++ b/OpenCL/m09800_a0.cl @@ -8,20 +8,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09800_a1.cl b/OpenCL/m09800_a1.cl index 1c11f01..48cc909 100644 --- a/OpenCL/m09800_a1.cl +++ b/OpenCL/m09800_a1.cl @@ -8,18 +8,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09800_a3.cl b/OpenCL/m09800_a3.cl index 6fc92c0..eaf1fa8 100644 --- a/OpenCL/m09800_a3.cl +++ b/OpenCL/m09800_a3.cl @@ -7,18 +7,18 @@ #define _OLDOFFICE34_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09810_a0.cl b/OpenCL/m09810_a0.cl index 40fb8a0..b27ee74 100644 --- a/OpenCL/m09810_a0.cl +++ b/OpenCL/m09810_a0.cl @@ -8,20 +8,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09810_a1.cl b/OpenCL/m09810_a1.cl index 527b1bd..bf1ae52 100644 --- a/OpenCL/m09810_a1.cl +++ b/OpenCL/m09810_a1.cl @@ -8,18 +8,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09810_a3.cl b/OpenCL/m09810_a3.cl index 5bfc896..56cf7c7 100644 --- a/OpenCL/m09810_a3.cl +++ b/OpenCL/m09810_a3.cl @@ -10,18 +10,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m09820_a0.cl b/OpenCL/m09820_a0.cl index 8e780bc..89dda1a 100644 --- a/OpenCL/m09820_a0.cl +++ b/OpenCL/m09820_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m09820_a1.cl b/OpenCL/m09820_a1.cl index f8025e5..b8075b3 100644 --- a/OpenCL/m09820_a1.cl +++ b/OpenCL/m09820_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m09820_a3.cl b/OpenCL/m09820_a3.cl index 3f8b332..0b9fc57 100644 --- a/OpenCL/m09820_a3.cl +++ b/OpenCL/m09820_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) { diff --git a/OpenCL/m09900_a0.cl b/OpenCL/m09900_a0.cl index 91ded72..fddcef8 100644 --- a/OpenCL/m09900_a0.cl +++ b/OpenCL/m09900_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m09900_m04 (__global pw_t *pws, __global kernel_rule_t * rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m09900_a1.cl b/OpenCL/m09900_a1.cl index e6f051d..0e63e55 100644 --- a/OpenCL/m09900_a1.cl +++ b/OpenCL/m09900_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m09900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m09900_a3.cl b/OpenCL/m09900_a3.cl index a11bf4c..30bae9b 100644 --- a/OpenCL/m09900_a3.cl +++ b/OpenCL/m09900_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m09900m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m10100_a0.cl b/OpenCL/m10100_a0.cl index 269aff3..ad82fb4 100644 --- a/OpenCL/m10100_a0.cl +++ b/OpenCL/m10100_a0.cl @@ -8,20 +8,20 @@ //incompatible to simd //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define SIPROUND(v0,v1,v2,v3) \ (v0) += (v1); \ diff --git a/OpenCL/m10100_a1.cl b/OpenCL/m10100_a1.cl index 345ad16..b797008 100644 --- a/OpenCL/m10100_a1.cl +++ b/OpenCL/m10100_a1.cl @@ -8,18 +8,18 @@ //incompatible to simd //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define SIPROUND(v0,v1,v2,v3) \ (v0) += (v1); \ diff --git a/OpenCL/m10100_a3.cl b/OpenCL/m10100_a3.cl index 116a7cf..ded29d3 100644 --- a/OpenCL/m10100_a3.cl +++ b/OpenCL/m10100_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define SIPROUND(v0,v1,v2,v3) \ (v0) += (v1); \ diff --git a/OpenCL/m10300.cl b/OpenCL/m10300.cl index 2493658..2dc8a99 100644 --- a/OpenCL/m10300.cl +++ b/OpenCL/m10300.cl @@ -5,20 +5,20 @@ #define _SHA1_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" void sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5]) { diff --git a/OpenCL/m10400_a0.cl b/OpenCL/m10400_a0.cl index 90cd0bc..9094a96 100644 --- a/OpenCL/m10400_a0.cl +++ b/OpenCL/m10400_a0.cl @@ -10,20 +10,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10400_a1.cl b/OpenCL/m10400_a1.cl index 093af29..df58488 100644 --- a/OpenCL/m10400_a1.cl +++ b/OpenCL/m10400_a1.cl @@ -10,18 +10,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10400_a3.cl b/OpenCL/m10400_a3.cl index 92ebfe5..b26f455 100644 --- a/OpenCL/m10400_a3.cl +++ b/OpenCL/m10400_a3.cl @@ -10,18 +10,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10410_a0.cl b/OpenCL/m10410_a0.cl index 518b84c..51b1d74 100644 --- a/OpenCL/m10410_a0.cl +++ b/OpenCL/m10410_a0.cl @@ -10,20 +10,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10410_a1.cl b/OpenCL/m10410_a1.cl index ec226b8..c5933ef 100644 --- a/OpenCL/m10410_a1.cl +++ b/OpenCL/m10410_a1.cl @@ -10,18 +10,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10410_a3.cl b/OpenCL/m10410_a3.cl index d43b657..b56c678 100644 --- a/OpenCL/m10410_a3.cl +++ b/OpenCL/m10410_a3.cl @@ -10,18 +10,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10420_a0.cl b/OpenCL/m10420_a0.cl index 9b69a25..f9d295b 100644 --- a/OpenCL/m10420_a0.cl +++ b/OpenCL/m10420_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10420_a1.cl b/OpenCL/m10420_a1.cl index 42cf320..9126e2c 100644 --- a/OpenCL/m10420_a1.cl +++ b/OpenCL/m10420_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10420_a3.cl b/OpenCL/m10420_a3.cl index 93b84c4..a7abb23 100644 --- a/OpenCL/m10420_a3.cl +++ b/OpenCL/m10420_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10500.cl b/OpenCL/m10500.cl index a24469f..721bfc5 100644 --- a/OpenCL/m10500.cl +++ b/OpenCL/m10500.cl @@ -5,20 +5,20 @@ #define _MD5_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 padding[8] = { diff --git a/OpenCL/m10700.cl b/OpenCL/m10700.cl index d606f2d..e0a7389 100644 --- a/OpenCL/m10700.cl +++ b/OpenCL/m10700.cl @@ -7,20 +7,20 @@ #define _PDF17L8_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" typedef struct { diff --git a/OpenCL/m10800_a0.cl b/OpenCL/m10800_a0.cl index 17f7540..c7de147 100644 --- a/OpenCL/m10800_a0.cl +++ b/OpenCL/m10800_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 6 #define DGST_R1 7 #define DGST_R2 4 #define DGST_R3 5 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u64 k_sha384[80] = { diff --git a/OpenCL/m10800_a1.cl b/OpenCL/m10800_a1.cl index bcb0c58..a3897b7 100644 --- a/OpenCL/m10800_a1.cl +++ b/OpenCL/m10800_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 6 #define DGST_R1 7 #define DGST_R2 4 #define DGST_R3 5 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha384[80] = { diff --git a/OpenCL/m10800_a3.cl b/OpenCL/m10800_a3.cl index 7bfc741..fae6a72 100644 --- a/OpenCL/m10800_a3.cl +++ b/OpenCL/m10800_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 6 #define DGST_R1 7 #define DGST_R2 4 #define DGST_R3 5 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u64 k_sha384[80] = { diff --git a/OpenCL/m10900.cl b/OpenCL/m10900.cl index 9427fab..275b049 100644 --- a/OpenCL/m10900.cl +++ b/OpenCL/m10900.cl @@ -7,21 +7,21 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m11000_a0.cl b/OpenCL/m11000_a0.cl index a209c8a..2b5970d 100644 --- a/OpenCL/m11000_a0.cl +++ b/OpenCL/m11000_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m11000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m11000_a1.cl b/OpenCL/m11000_a1.cl index f6c8e7a..808914e 100644 --- a/OpenCL/m11000_a1.cl +++ b/OpenCL/m11000_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m11000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m11000_a3.cl b/OpenCL/m11000_a3.cl index 0c42ddf..569d149 100644 --- a/OpenCL/m11000_a3.cl +++ b/OpenCL/m11000_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m11000m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m11100_a0.cl b/OpenCL/m11100_a0.cl index 8ab430a..ec1f0fc 100644 --- a/OpenCL/m11100_a0.cl +++ b/OpenCL/m11100_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m11100_a1.cl b/OpenCL/m11100_a1.cl index 7ec184a..50ec905 100644 --- a/OpenCL/m11100_a1.cl +++ b/OpenCL/m11100_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m11100_a3.cl b/OpenCL/m11100_a3.cl index 2d00c49..a5d860c 100644 --- a/OpenCL/m11100_a3.cl +++ b/OpenCL/m11100_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m11200_a0.cl b/OpenCL/m11200_a0.cl index 1e39af0..f63ec3b 100644 --- a/OpenCL/m11200_a0.cl +++ b/OpenCL/m11200_a0.cl @@ -7,20 +7,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m11200_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m11200_a1.cl b/OpenCL/m11200_a1.cl index 062ec3b..0790a46 100644 --- a/OpenCL/m11200_a1.cl +++ b/OpenCL/m11200_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m11200_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m11200_a3.cl b/OpenCL/m11200_a3.cl index eec3317..4d1ecfb 100644 --- a/OpenCL/m11200_a3.cl +++ b/OpenCL/m11200_a3.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m11200m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m11300.cl b/OpenCL/m11300.cl index 3bfa242..df9fdd5 100644 --- a/OpenCL/m11300.cl +++ b/OpenCL/m11300.cl @@ -7,17 +7,17 @@ #define _SHA512_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m11400_a0.cl b/OpenCL/m11400_a0.cl index 5ac6101..2d3981b 100644 --- a/OpenCL/m11400_a0.cl +++ b/OpenCL/m11400_a0.cl @@ -10,20 +10,20 @@ //incompatible because of brances //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m11400_a1.cl b/OpenCL/m11400_a1.cl index d16b4ee..c15333d 100644 --- a/OpenCL/m11400_a1.cl +++ b/OpenCL/m11400_a1.cl @@ -8,18 +8,18 @@ //incompatible because of brances //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m11400_a3.cl b/OpenCL/m11400_a3.cl index 8086990..4caa348 100644 --- a/OpenCL/m11400_a3.cl +++ b/OpenCL/m11400_a3.cl @@ -11,18 +11,18 @@ //incompatible because of brances //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 3 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m11500_a0.cl b/OpenCL/m11500_a0.cl index 3bcc15b..088881e 100644 --- a/OpenCL/m11500_a0.cl +++ b/OpenCL/m11500_a0.cl @@ -10,20 +10,20 @@ //incompatible because of branches //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __constant u32 crc32tab[0x100] = { diff --git a/OpenCL/m11500_a1.cl b/OpenCL/m11500_a1.cl index 3dd6c46..6d3bb7f 100644 --- a/OpenCL/m11500_a1.cl +++ b/OpenCL/m11500_a1.cl @@ -8,18 +8,18 @@ //incompatible because of branches //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 crc32tab[0x100] = { diff --git a/OpenCL/m11500_a3.cl b/OpenCL/m11500_a3.cl index c8e9986..4af75ed 100644 --- a/OpenCL/m11500_a3.cl +++ b/OpenCL/m11500_a3.cl @@ -10,18 +10,18 @@ //incompatible because of branches //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __constant u32 crc32tab[0x100] = { diff --git a/OpenCL/m11600.cl b/OpenCL/m11600.cl index c7af848..00ae013 100644 --- a/OpenCL/m11600.cl +++ b/OpenCL/m11600.cl @@ -7,20 +7,20 @@ #define _SEVEN_ZIP_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m11700_a0.cl b/OpenCL/m11700_a0.cl index 2e93185..db98b14 100644 --- a/OpenCL/m11700_a0.cl +++ b/OpenCL/m11700_a0.cl @@ -11,20 +11,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define INITVAL 0x0101010101010101 diff --git a/OpenCL/m11700_a1.cl b/OpenCL/m11700_a1.cl index 36ce0b9..e0320dd 100644 --- a/OpenCL/m11700_a1.cl +++ b/OpenCL/m11700_a1.cl @@ -10,18 +10,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define INITVAL 0x0101010101010101 diff --git a/OpenCL/m11700_a3.cl b/OpenCL/m11700_a3.cl index 1569b1f..0224f23 100644 --- a/OpenCL/m11700_a3.cl +++ b/OpenCL/m11700_a3.cl @@ -11,18 +11,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define INITVAL 0x0101010101010101 diff --git a/OpenCL/m11800_a0.cl b/OpenCL/m11800_a0.cl index f282c3e..87e63b5 100644 --- a/OpenCL/m11800_a0.cl +++ b/OpenCL/m11800_a0.cl @@ -11,20 +11,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #define INITVAL 0 diff --git a/OpenCL/m11800_a1.cl b/OpenCL/m11800_a1.cl index cf8392c..0cd6cf8 100644 --- a/OpenCL/m11800_a1.cl +++ b/OpenCL/m11800_a1.cl @@ -10,18 +10,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define INITVAL 0 diff --git a/OpenCL/m11800_a3.cl b/OpenCL/m11800_a3.cl index 2370800..4ef9bb0 100644 --- a/OpenCL/m11800_a3.cl +++ b/OpenCL/m11800_a3.cl @@ -11,18 +11,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #define INITVAL 0 diff --git a/OpenCL/m11900.cl b/OpenCL/m11900.cl index 53eec8f..ffa0f6c 100644 --- a/OpenCL/m11900.cl +++ b/OpenCL/m11900.cl @@ -7,21 +7,21 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" void md5_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) { diff --git a/OpenCL/m12000.cl b/OpenCL/m12000.cl index ffbcf86..0619bcd 100644 --- a/OpenCL/m12000.cl +++ b/OpenCL/m12000.cl @@ -7,21 +7,21 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" void sha1_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5]) { diff --git a/OpenCL/m12200.cl b/OpenCL/m12200.cl index 7b789bc..13af738 100644 --- a/OpenCL/m12200.cl +++ b/OpenCL/m12200.cl @@ -5,20 +5,20 @@ #define _SHA512_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m12300.cl b/OpenCL/m12300.cl index 5430b4f..3a87faa 100644 --- a/OpenCL/m12300.cl +++ b/OpenCL/m12300.cl @@ -5,20 +5,20 @@ #define _PBKDF2_SHA512_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u64 k_sha512[80] = { diff --git a/OpenCL/m12400.cl b/OpenCL/m12400.cl index 7902c5d..d32f9b0 100644 --- a/OpenCL/m12400.cl +++ b/OpenCL/m12400.cl @@ -7,20 +7,20 @@ #define _DES_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" #define PERM_OP(a,b,tt,n,m) \ { \ diff --git a/OpenCL/m12500.cl b/OpenCL/m12500.cl index 3292bc1..8c6b6e3 100644 --- a/OpenCL/m12500.cl +++ b/OpenCL/m12500.cl @@ -7,21 +7,21 @@ #define _RAR3_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" +#include "inc_hash_functions.cl" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" #define ROUNDS 0x40000 diff --git a/OpenCL/m12600_a0.cl b/OpenCL/m12600_a0.cl index d9fe579..a6368f4 100644 --- a/OpenCL/m12600_a0.cl +++ b/OpenCL/m12600_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m12600_a1.cl b/OpenCL/m12600_a1.cl index 9b527c9..14f9018 100644 --- a/OpenCL/m12600_a1.cl +++ b/OpenCL/m12600_a1.cl @@ -7,18 +7,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m12600_a3.cl b/OpenCL/m12600_a3.cl index dc8b835..dd6c899 100644 --- a/OpenCL/m12600_a3.cl +++ b/OpenCL/m12600_a3.cl @@ -10,18 +10,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 7 #define DGST_R2 2 #define DGST_R3 6 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" #if VECT_SIZE == 1 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) diff --git a/OpenCL/m12700.cl b/OpenCL/m12700.cl index 1e51e35..4175979 100644 --- a/OpenCL/m12700.cl +++ b/OpenCL/m12700.cl @@ -7,20 +7,20 @@ #define _SHA1_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m12800.cl b/OpenCL/m12800.cl index a9e3f93..d138fc7 100644 --- a/OpenCL/m12800.cl +++ b/OpenCL/m12800.cl @@ -5,21 +5,21 @@ #define _MS_DRSR_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" +#include "inc_hash_functions.cl" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" #define uint_to_hex_lower8(i) l_bin2asc[(i)] diff --git a/OpenCL/m12900.cl b/OpenCL/m12900.cl index 817a819..589c170 100644 --- a/OpenCL/m12900.cl +++ b/OpenCL/m12900.cl @@ -5,21 +5,21 @@ #define _ANDROIDFDE_SAMSUNG_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" +#include "inc_hash_functions.cl" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m13000.cl b/OpenCL/m13000.cl index 267081c..c07d8da 100644 --- a/OpenCL/m13000.cl +++ b/OpenCL/m13000.cl @@ -5,21 +5,21 @@ #define _RAR5_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" +#include "inc_hash_functions.cl" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m13100_a0.cl b/OpenCL/m13100_a0.cl index bd16342..3544eaa 100644 --- a/OpenCL/m13100_a0.cl +++ b/OpenCL/m13100_a0.cl @@ -11,20 +11,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m13100_a1.cl b/OpenCL/m13100_a1.cl index c49f2fd..2aba9d2 100644 --- a/OpenCL/m13100_a1.cl +++ b/OpenCL/m13100_a1.cl @@ -11,18 +11,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m13100_a3.cl b/OpenCL/m13100_a3.cl index b549673..6113ae1 100644 --- a/OpenCL/m13100_a3.cl +++ b/OpenCL/m13100_a3.cl @@ -11,18 +11,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" typedef struct { diff --git a/OpenCL/m13200.cl b/OpenCL/m13200.cl index 6027cca..a104b16 100644 --- a/OpenCL/m13200.cl +++ b/OpenCL/m13200.cl @@ -7,17 +7,17 @@ #define _SHA1_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m13300_a0.cl b/OpenCL/m13300_a0.cl index 5ac204b..bd45d38 100644 --- a/OpenCL/m13300_a0.cl +++ b/OpenCL/m13300_a0.cl @@ -9,20 +9,20 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 4 #define DGST_R2 3 #define DGST_R3 2 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" __kernel void m13300_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m13300_a1.cl b/OpenCL/m13300_a1.cl index 39421ec..8f1ec30 100644 --- a/OpenCL/m13300_a1.cl +++ b/OpenCL/m13300_a1.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 4 #define DGST_R2 3 #define DGST_R3 2 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" __kernel void m13300_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { diff --git a/OpenCL/m13300_a3.cl b/OpenCL/m13300_a3.cl index dfeff7b..7109f7b 100644 --- a/OpenCL/m13300_a3.cl +++ b/OpenCL/m13300_a3.cl @@ -9,18 +9,18 @@ #define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 4 #define DGST_R2 3 #define DGST_R3 2 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void m13300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) { diff --git a/OpenCL/m13400.cl b/OpenCL/m13400.cl index e0ec6d9..6079f55 100644 --- a/OpenCL/m13400.cl +++ b/OpenCL/m13400.cl @@ -7,19 +7,19 @@ #define _KEEPASS_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_twofish256.c" +#include "inc_cipher_twofish256.cl" __constant u32 te0[256] = { diff --git a/OpenCL/m13500_a0.cl b/OpenCL/m13500_a0.cl index 9538b46..9339212 100644 --- a/OpenCL/m13500_a0.cl +++ b/OpenCL/m13500_a0.cl @@ -8,20 +8,20 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "include/rp_kernel.h" -#include "OpenCL/rp.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" void memcat64c_be (u32x block[16], const u32 offset, u32x carry[16]) { diff --git a/OpenCL/m13500_a1.cl b/OpenCL/m13500_a1.cl index c1e3db1..b07279b 100644 --- a/OpenCL/m13500_a1.cl +++ b/OpenCL/m13500_a1.cl @@ -8,18 +8,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void memcat64c_be (u32x block[16], const u32 offset, u32x carry[16]) { diff --git a/OpenCL/m13500_a3.cl b/OpenCL/m13500_a3.cl index 046db7e..28d7427 100644 --- a/OpenCL/m13500_a3.cl +++ b/OpenCL/m13500_a3.cl @@ -8,18 +8,18 @@ //too much register pressure //#define NEW_SIMD_CODE -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 3 #define DGST_R1 4 #define DGST_R2 2 #define DGST_R3 1 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" -#include "OpenCL/simd.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" void memcat64c_be (u32x block[16], const u32 offset, u32x carry[16]) { diff --git a/OpenCL/m13600.cl b/OpenCL/m13600.cl index 5499f40..5cd9c6d 100644 --- a/OpenCL/m13600.cl +++ b/OpenCL/m13600.cl @@ -5,21 +5,21 @@ #define _ZIP2_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" +#include "inc_hash_functions.cl" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_types.cl" +#include "inc_common.cl" -#define COMPARE_S "OpenCL/check_single_comp4.c" -#define COMPARE_M "OpenCL/check_multi_comp4.c" +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" void sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5]) { diff --git a/OpenCL/m13751.cl b/OpenCL/m13751.cl index 06e3fab..f375323 100644 --- a/OpenCL/m13751.cl +++ b/OpenCL/m13751.cl @@ -5,21 +5,21 @@ #define _SHA256_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m13752.cl b/OpenCL/m13752.cl index 0613321..2f7a9a5 100644 --- a/OpenCL/m13752.cl +++ b/OpenCL/m13752.cl @@ -5,21 +5,21 @@ #define _SHA256_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/m13753.cl b/OpenCL/m13753.cl index be1a9e9..489901c 100644 --- a/OpenCL/m13753.cl +++ b/OpenCL/m13753.cl @@ -5,21 +5,21 @@ #define _SHA256_ -#include "include/constants.h" -#include "include/kernel_vendor.h" +#include "inc_hash_constants.h" +#include "inc_vendor.cl" #define DGST_R0 0 #define DGST_R1 1 #define DGST_R2 2 #define DGST_R3 3 -#include "include/kernel_functions.c" -#include "OpenCL/types_ocl.c" -#include "OpenCL/common.c" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" -#include "OpenCL/kernel_aes256.c" -#include "OpenCL/kernel_twofish256.c" -#include "OpenCL/kernel_serpent256.c" +#include "inc_cipher_aes256.cl" +#include "inc_cipher_twofish256.cl" +#include "inc_cipher_serpent256.cl" __constant u32 k_sha256[64] = { diff --git a/OpenCL/markov_be.cl b/OpenCL/markov_be.cl index a429b2e..9d0720f 100644 --- a/OpenCL/markov_be.cl +++ b/OpenCL/markov_be.cl @@ -3,11 +3,11 @@ * License.....: MIT */ -#include "include/kernel_vendor.h" +#include "inc_vendor.cl" #define CHARSIZ 256 -#include "OpenCL/types_ocl.c" +#include "inc_types.cl" inline void generate_pw (u32 pw_buf[16], __global cs_t *root_css_buf, __global cs_t *markov_css_buf, const u32 pw_l_len, const u32 pw_r_len, const u32 mask80, const u32 bits14, const u32 bits15, u64 val) { diff --git a/OpenCL/markov_le.cl b/OpenCL/markov_le.cl index dc0001c..2729446 100644 --- a/OpenCL/markov_le.cl +++ b/OpenCL/markov_le.cl @@ -3,11 +3,11 @@ * License.....: MIT */ -#include "include/kernel_vendor.h" +#include "inc_vendor.cl" #define CHARSIZ 256 -#include "OpenCL/types_ocl.c" +#include "inc_types.cl" inline void generate_pw (u32 pw_buf[16], __global cs_t *root_css_buf, __global cs_t *markov_css_buf, const u32 pw_l_len, const u32 pw_r_len, const u32 mask80, const u32 bits14, const u32 bits15, u64 val) { diff --git a/OpenCL/rp.c b/OpenCL/rp.c deleted file mode 100644 index 3a93e19..0000000 --- a/OpenCL/rp.c +++ /dev/null @@ -1,4357 +0,0 @@ -/** - * Authors.....: Jens Steube - * magnum - * - * License.....: MIT - */ - -inline u32 apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len); -inline u32 apply_rules (const __global u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len); -inline u32x apply_rules_vect (const u32 pw_buf0[4], const u32 pw_buf1[4], const u32 pw_len, const __global kernel_rule_t *rules_buf, const u32 il_pos, u32x w0[4], u32x w1[4]); - -inline u32 generate_cmask (u32 buf) -{ - const u32 rmask = ((buf & 0x40404040) >> 1) - & ~((buf & 0x80808080) >> 2); - - const u32 hmask = (buf & 0x1f1f1f1f) + 0x05050505; - const u32 lmask = (buf & 0x1f1f1f1f) + 0x1f1f1f1f; - - return rmask & ~hmask & lmask; -} - -inline void truncate_right (u32 w0[4], u32 w1[4], const u32 len) -{ - const u32 tmp = (1 << ((len % 4) * 8)) - 1; - - switch (len / 4) - { - case 0: w0[0] &= tmp; - w0[1] = 0; - w0[2] = 0; - w0[3] = 0; - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - break; - case 1: w0[1] &= tmp; - w0[2] = 0; - w0[3] = 0; - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - break; - case 2: w0[2] &= tmp; - w0[3] = 0; - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - break; - case 3: w0[3] &= tmp; - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - break; - case 4: w1[0] &= tmp; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - break; - case 5: w1[1] &= tmp; - w1[2] = 0; - w1[3] = 0; - break; - case 6: w1[2] &= tmp; - w1[3] = 0; - break; - case 7: w1[3] &= tmp; - break; - } -} - -inline void truncate_left (u32 w0[4], u32 w1[4], const u32 len) -{ - const u32 tmp = ~((1 << ((len % 4) * 8)) - 1); - - switch (len / 4) - { - case 0: w0[0] &= tmp; - break; - case 1: w0[0] = 0; - w0[1] &= tmp; - break; - case 2: w0[0] = 0; - w0[1] = 0; - w0[2] &= tmp; - break; - case 3: w0[0] = 0; - w0[1] = 0; - w0[2] = 0; - w0[3] &= tmp; - break; - case 4: w0[0] = 0; - w0[1] = 0; - w0[2] = 0; - w0[3] = 0; - w1[0] &= tmp; - break; - case 5: w0[0] = 0; - w0[1] = 0; - w0[2] = 0; - w0[3] = 0; - w1[0] = 0; - w1[1] &= tmp; - break; - case 6: w0[0] = 0; - w0[1] = 0; - w0[2] = 0; - w0[3] = 0; - w1[0] = 0; - w1[1] = 0; - w1[2] &= tmp; - break; - case 7: w0[0] = 0; - w0[1] = 0; - w0[2] = 0; - w0[3] = 0; - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] &= tmp; - break; - } -} - -inline void lshift_block (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4]) -{ - #ifdef IS_NV - out0[0] = __byte_perm_S (in0[0], in0[1], 0x4321); - out0[1] = __byte_perm_S (in0[1], in0[2], 0x4321); - out0[2] = __byte_perm_S (in0[2], in0[3], 0x4321); - out0[3] = __byte_perm_S (in0[3], in1[0], 0x4321); - out1[0] = __byte_perm_S (in1[0], in1[1], 0x4321); - out1[1] = __byte_perm_S (in1[1], in1[2], 0x4321); - out1[2] = __byte_perm_S (in1[2], in1[3], 0x4321); - out1[3] = __byte_perm_S (in1[3], 0, 0x4321); - #endif - - #if defined IS_AMD || defined IS_GENERIC - out0[0] = amd_bytealign_S (in0[1], in0[0], 1); - out0[1] = amd_bytealign_S (in0[2], in0[1], 1); - out0[2] = amd_bytealign_S (in0[3], in0[2], 1); - out0[3] = amd_bytealign_S (in1[0], in0[3], 1); - out1[0] = amd_bytealign_S (in1[1], in1[0], 1); - out1[1] = amd_bytealign_S (in1[2], in1[1], 1); - out1[2] = amd_bytealign_S (in1[3], in1[2], 1); - out1[3] = amd_bytealign_S ( 0, in1[3], 1); - #endif -} - -inline void rshift_block (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4]) -{ - #ifdef IS_NV - out1[3] = __byte_perm_S (in1[2], in1[3], 0x6543); - out1[2] = __byte_perm_S (in1[1], in1[2], 0x6543); - out1[1] = __byte_perm_S (in1[0], in1[1], 0x6543); - out1[0] = __byte_perm_S (in0[3], in1[0], 0x6543); - out0[3] = __byte_perm_S (in0[2], in0[3], 0x6543); - out0[2] = __byte_perm_S (in0[1], in0[2], 0x6543); - out0[1] = __byte_perm_S (in0[0], in0[1], 0x6543); - out0[0] = __byte_perm_S ( 0, in0[0], 0x6543); - #endif - - #if defined IS_AMD || defined IS_GENERIC - out1[3] = amd_bytealign_S (in1[3], in1[2], 3); - out1[2] = amd_bytealign_S (in1[2], in1[1], 3); - out1[1] = amd_bytealign_S (in1[1], in1[0], 3); - out1[0] = amd_bytealign_S (in1[0], in0[3], 3); - out0[3] = amd_bytealign_S (in0[3], in0[2], 3); - out0[2] = amd_bytealign_S (in0[2], in0[1], 3); - out0[1] = amd_bytealign_S (in0[1], in0[0], 3); - out0[0] = amd_bytealign_S (in0[0], 0, 3); - #endif -} - -inline void lshift_block_N (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4], const u32 num) -{ - #ifdef IS_NV - switch (num) - { - case 0: out0[0] = in0[0]; - out0[1] = in0[1]; - out0[2] = in0[2]; - out0[3] = in0[3]; - out1[0] = in1[0]; - out1[1] = in1[1]; - out1[2] = in1[2]; - out1[3] = in1[3]; - break; - case 1: out0[0] = __byte_perm_S (in0[0], in0[1], 0x4321); - out0[1] = __byte_perm_S (in0[1], in0[2], 0x4321); - out0[2] = __byte_perm_S (in0[2], in0[3], 0x4321); - out0[3] = __byte_perm_S (in0[3], in1[0], 0x4321); - out1[0] = __byte_perm_S (in1[0], in1[1], 0x4321); - out1[1] = __byte_perm_S (in1[1], in1[2], 0x4321); - out1[2] = __byte_perm_S (in1[2], in1[3], 0x4321); - out1[3] = __byte_perm_S (in1[3], 0, 0x4321); - break; - case 2: out0[0] = __byte_perm_S (in0[0], in0[1], 0x5432); - out0[1] = __byte_perm_S (in0[1], in0[2], 0x5432); - out0[2] = __byte_perm_S (in0[2], in0[3], 0x5432); - out0[3] = __byte_perm_S (in0[3], in1[0], 0x5432); - out1[0] = __byte_perm_S (in1[0], in1[1], 0x5432); - out1[1] = __byte_perm_S (in1[1], in1[2], 0x5432); - out1[2] = __byte_perm_S (in1[2], in1[3], 0x5432); - out1[3] = __byte_perm_S (in1[3], 0, 0x5432); - break; - case 3: out0[0] = __byte_perm_S (in0[0], in0[1], 0x6543); - out0[1] = __byte_perm_S (in0[1], in0[2], 0x6543); - out0[2] = __byte_perm_S (in0[2], in0[3], 0x6543); - out0[3] = __byte_perm_S (in0[3], in1[0], 0x6543); - out1[0] = __byte_perm_S (in1[0], in1[1], 0x6543); - out1[1] = __byte_perm_S (in1[1], in1[2], 0x6543); - out1[2] = __byte_perm_S (in1[2], in1[3], 0x6543); - out1[3] = __byte_perm_S (in1[3], 0, 0x6543); - break; - case 4: out0[0] = in0[1]; - out0[1] = in0[2]; - out0[2] = in0[3]; - out0[3] = in1[0]; - out1[0] = in1[1]; - out1[1] = in1[2]; - out1[2] = in1[3]; - out1[3] = 0; - break; - case 5: out0[0] = __byte_perm_S (in0[1], in0[2], 0x4321); - out0[1] = __byte_perm_S (in0[2], in0[3], 0x4321); - out0[2] = __byte_perm_S (in0[3], in1[0], 0x4321); - out0[3] = __byte_perm_S (in1[0], in1[1], 0x4321); - out1[0] = __byte_perm_S (in1[1], in1[2], 0x4321); - out1[1] = __byte_perm_S (in1[2], in1[3], 0x4321); - out1[2] = __byte_perm_S (in1[3], 0, 0x4321); - out1[3] = 0; - break; - case 6: out0[0] = __byte_perm_S (in0[1], in0[2], 0x5432); - out0[1] = __byte_perm_S (in0[2], in0[3], 0x5432); - out0[2] = __byte_perm_S (in0[3], in1[0], 0x5432); - out0[3] = __byte_perm_S (in1[0], in1[1], 0x5432); - out1[0] = __byte_perm_S (in1[1], in1[2], 0x5432); - out1[1] = __byte_perm_S (in1[2], in1[3], 0x5432); - out1[2] = __byte_perm_S (in1[3], 0, 0x5432); - out1[3] = 0; - break; - case 7: out0[0] = __byte_perm_S (in0[1], in0[2], 0x6543); - out0[1] = __byte_perm_S (in0[2], in0[3], 0x6543); - out0[2] = __byte_perm_S (in0[3], in1[0], 0x6543); - out0[3] = __byte_perm_S (in1[0], in1[1], 0x6543); - out1[0] = __byte_perm_S (in1[1], in1[2], 0x6543); - out1[1] = __byte_perm_S (in1[2], in1[3], 0x6543); - out1[2] = __byte_perm_S (in1[3], 0, 0x6543); - out1[3] = 0; - break; - case 8: out0[0] = in0[2]; - out0[1] = in0[3]; - out0[2] = in1[0]; - out0[3] = in1[1]; - out1[0] = in1[2]; - out1[1] = in1[3]; - out1[2] = 0; - out1[3] = 0; - break; - case 9: out0[0] = __byte_perm_S (in0[2], in0[3], 0x4321); - out0[1] = __byte_perm_S (in0[3], in1[0], 0x4321); - out0[2] = __byte_perm_S (in1[0], in1[1], 0x4321); - out0[3] = __byte_perm_S (in1[1], in1[2], 0x4321); - out1[0] = __byte_perm_S (in1[2], in1[3], 0x4321); - out1[1] = __byte_perm_S (in1[3], 0, 0x4321); - out1[2] = 0; - out1[3] = 0; - break; - case 10: out0[0] = __byte_perm_S (in0[2], in0[3], 0x5432); - out0[1] = __byte_perm_S (in0[3], in1[0], 0x5432); - out0[2] = __byte_perm_S (in1[0], in1[1], 0x5432); - out0[3] = __byte_perm_S (in1[1], in1[2], 0x5432); - out1[0] = __byte_perm_S (in1[2], in1[3], 0x5432); - out1[1] = __byte_perm_S (in1[3], 0, 0x5432); - out1[2] = 0; - out1[3] = 0; - break; - case 11: out0[0] = __byte_perm_S (in0[2], in0[3], 0x6543); - out0[1] = __byte_perm_S (in0[3], in1[0], 0x6543); - out0[2] = __byte_perm_S (in1[0], in1[1], 0x6543); - out0[3] = __byte_perm_S (in1[1], in1[2], 0x6543); - out1[0] = __byte_perm_S (in1[2], in1[3], 0x6543); - out1[1] = __byte_perm_S (in1[3], 0, 0x6543); - out1[2] = 0; - out1[3] = 0; - break; - case 12: out0[0] = in0[3]; - out0[1] = in1[0]; - out0[2] = in1[1]; - out0[3] = in1[2]; - out1[0] = in1[3]; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 13: - out0[0] = __byte_perm_S (in0[3], in1[0], 0x4321); - out0[1] = __byte_perm_S (in1[0], in1[1], 0x4321); - out0[2] = __byte_perm_S (in1[1], in1[2], 0x4321); - out0[3] = __byte_perm_S (in1[2], in1[3], 0x4321); - out1[0] = __byte_perm_S (in1[3], 0, 0x4321); - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 14: out0[0] = __byte_perm_S (in0[3], in1[0], 0x5432); - out0[1] = __byte_perm_S (in1[0], in1[1], 0x5432); - out0[2] = __byte_perm_S (in1[1], in1[2], 0x5432); - out0[3] = __byte_perm_S (in1[2], in1[3], 0x5432); - out1[0] = __byte_perm_S (in1[3], 0, 0x5432); - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 15: out0[0] = __byte_perm_S (in0[3], in1[0], 0x6543); - out0[1] = __byte_perm_S (in1[0], in1[1], 0x6543); - out0[2] = __byte_perm_S (in1[1], in1[2], 0x6543); - out0[3] = __byte_perm_S (in1[2], in1[3], 0x6543); - out1[0] = __byte_perm_S (in1[3], 0, 0x6543); - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 16: out0[0] = in1[0]; - out0[1] = in1[1]; - out0[2] = in1[2]; - out0[3] = in1[3]; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 17: out0[0] = __byte_perm_S (in1[0], in1[1], 0x4321); - out0[1] = __byte_perm_S (in1[1], in1[2], 0x4321); - out0[2] = __byte_perm_S (in1[2], in1[3], 0x4321); - out0[3] = __byte_perm_S (in1[3], 0, 0x4321); - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 18: out0[0] = __byte_perm_S (in1[0], in1[1], 0x5432); - out0[1] = __byte_perm_S (in1[1], in1[2], 0x5432); - out0[2] = __byte_perm_S (in1[2], in1[3], 0x5432); - out0[3] = __byte_perm_S (in1[3], 0, 0x5432); - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 19: out0[0] = __byte_perm_S (in1[0], in1[1], 0x6543); - out0[1] = __byte_perm_S (in1[1], in1[2], 0x6543); - out0[2] = __byte_perm_S (in1[2], in1[3], 0x6543); - out0[3] = __byte_perm_S (in1[3], 0, 0x6543); - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 20: out0[0] = in1[1]; - out0[1] = in1[2]; - out0[2] = in1[3]; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 21: out0[0] = __byte_perm_S (in1[1], in1[2], 0x4321); - out0[1] = __byte_perm_S (in1[2], in1[3], 0x4321); - out0[2] = __byte_perm_S (in1[3], 0, 0x4321); - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 22: out0[0] = __byte_perm_S (in1[1], in1[2], 0x5432); - out0[1] = __byte_perm_S (in1[2], in1[3], 0x5432); - out0[2] = __byte_perm_S (in1[3], 0, 0x5432); - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 23: out0[0] = __byte_perm_S (in1[1], in1[2], 0x6543); - out0[1] = __byte_perm_S (in1[2], in1[3], 0x6543); - out0[2] = __byte_perm_S (in1[3], 0, 0x6543); - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 24: out0[0] = in1[2]; - out0[1] = in1[3]; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 25: out0[0] = __byte_perm_S (in1[2], in1[3], 0x4321); - out0[1] = __byte_perm_S (in1[3], 0, 0x4321); - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 26: out0[0] = __byte_perm_S (in1[2], in1[3], 0x5432); - out0[1] = __byte_perm_S (in1[3], 0, 0x5432); - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 27: out0[0] = __byte_perm_S (in1[2], in1[3], 0x6543); - out0[1] = __byte_perm_S (in1[3], 0, 0x6543); - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 28: out0[0] = in1[3]; - out0[1] = 0; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 29: out0[0] = __byte_perm_S (in1[3], 0, 0x4321); - out0[1] = 0; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 30: out0[0] = __byte_perm_S (in1[3], 0, 0x5432); - out0[1] = 0; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 31: out0[0] = __byte_perm_S (in1[3], 0, 0x6543); - out0[1] = 0; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - } - #endif - - #if defined IS_AMD || defined IS_GENERIC - switch (num) - { - case 0: out0[0] = in0[0]; - out0[1] = in0[1]; - out0[2] = in0[2]; - out0[3] = in0[3]; - out1[0] = in1[0]; - out1[1] = in1[1]; - out1[2] = in1[2]; - out1[3] = in1[3]; - break; - case 1: out0[0] = amd_bytealign_S (in0[1], in0[0], 1); - out0[1] = amd_bytealign_S (in0[2], in0[1], 1); - out0[2] = amd_bytealign_S (in0[3], in0[2], 1); - out0[3] = amd_bytealign_S (in1[0], in0[3], 1); - out1[0] = amd_bytealign_S (in1[1], in1[0], 1); - out1[1] = amd_bytealign_S (in1[2], in1[1], 1); - out1[2] = amd_bytealign_S (in1[3], in1[2], 1); - out1[3] = amd_bytealign_S ( 0, in1[3], 1); - break; - case 2: out0[0] = amd_bytealign_S (in0[1], in0[0], 2); - out0[1] = amd_bytealign_S (in0[2], in0[1], 2); - out0[2] = amd_bytealign_S (in0[3], in0[2], 2); - out0[3] = amd_bytealign_S (in1[0], in0[3], 2); - out1[0] = amd_bytealign_S (in1[1], in1[0], 2); - out1[1] = amd_bytealign_S (in1[2], in1[1], 2); - out1[2] = amd_bytealign_S (in1[3], in1[2], 2); - out1[3] = amd_bytealign_S ( 0, in1[3], 2); - break; - case 3: out0[0] = amd_bytealign_S (in0[1], in0[0], 3); - out0[1] = amd_bytealign_S (in0[2], in0[1], 3); - out0[2] = amd_bytealign_S (in0[3], in0[2], 3); - out0[3] = amd_bytealign_S (in1[0], in0[3], 3); - out1[0] = amd_bytealign_S (in1[1], in1[0], 3); - out1[1] = amd_bytealign_S (in1[2], in1[1], 3); - out1[2] = amd_bytealign_S (in1[3], in1[2], 3); - out1[3] = amd_bytealign_S ( 0, in1[3], 3); - break; - case 4: out0[0] = in0[1]; - out0[1] = in0[2]; - out0[2] = in0[3]; - out0[3] = in1[0]; - out1[0] = in1[1]; - out1[1] = in1[2]; - out1[2] = in1[3]; - out1[3] = 0; - break; - case 5: out0[0] = amd_bytealign_S (in0[2], in0[1], 1); - out0[1] = amd_bytealign_S (in0[3], in0[2], 1); - out0[2] = amd_bytealign_S (in1[0], in0[3], 1); - out0[3] = amd_bytealign_S (in1[1], in1[0], 1); - out1[0] = amd_bytealign_S (in1[2], in1[1], 1); - out1[1] = amd_bytealign_S (in1[3], in1[2], 1); - out1[2] = amd_bytealign_S ( 0, in1[3], 1); - out1[3] = 0; - break; - case 6: out0[0] = amd_bytealign_S (in0[2], in0[1], 2); - out0[1] = amd_bytealign_S (in0[3], in0[2], 2); - out0[2] = amd_bytealign_S (in1[0], in0[3], 2); - out0[3] = amd_bytealign_S (in1[1], in1[0], 2); - out1[0] = amd_bytealign_S (in1[2], in1[1], 2); - out1[1] = amd_bytealign_S (in1[3], in1[2], 2); - out1[2] = amd_bytealign_S ( 0, in1[3], 2); - out1[3] = 0; - break; - case 7: out0[0] = amd_bytealign_S (in0[2], in0[1], 3); - out0[1] = amd_bytealign_S (in0[3], in0[2], 3); - out0[2] = amd_bytealign_S (in1[0], in0[3], 3); - out0[3] = amd_bytealign_S (in1[1], in1[0], 3); - out1[0] = amd_bytealign_S (in1[2], in1[1], 3); - out1[1] = amd_bytealign_S (in1[3], in1[2], 3); - out1[2] = amd_bytealign_S ( 0, in1[3], 3); - out1[3] = 0; - break; - case 8: out0[0] = in0[2]; - out0[1] = in0[3]; - out0[2] = in1[0]; - out0[3] = in1[1]; - out1[0] = in1[2]; - out1[1] = in1[3]; - out1[2] = 0; - out1[3] = 0; - break; - case 9: out0[0] = amd_bytealign_S (in0[3], in0[2], 1); - out0[1] = amd_bytealign_S (in1[0], in0[3], 1); - out0[2] = amd_bytealign_S (in1[1], in1[0], 1); - out0[3] = amd_bytealign_S (in1[2], in1[1], 1); - out1[0] = amd_bytealign_S (in1[3], in1[2], 1); - out1[1] = amd_bytealign_S ( 0, in1[3], 1); - out1[2] = 0; - out1[3] = 0; - break; - case 10: out0[0] = amd_bytealign_S (in0[3], in0[2], 2); - out0[1] = amd_bytealign_S (in1[0], in0[3], 2); - out0[2] = amd_bytealign_S (in1[1], in1[0], 2); - out0[3] = amd_bytealign_S (in1[2], in1[1], 2); - out1[0] = amd_bytealign_S (in1[3], in1[2], 2); - out1[1] = amd_bytealign_S ( 0, in1[3], 2); - out1[2] = 0; - out1[3] = 0; - break; - case 11: out0[0] = amd_bytealign_S (in0[3], in0[2], 3); - out0[1] = amd_bytealign_S (in1[0], in0[3], 3); - out0[2] = amd_bytealign_S (in1[1], in1[0], 3); - out0[3] = amd_bytealign_S (in1[2], in1[1], 3); - out1[0] = amd_bytealign_S (in1[3], in1[2], 3); - out1[1] = amd_bytealign_S ( 0, in1[3], 3); - out1[2] = 0; - out1[3] = 0; - break; - case 12: out0[0] = in0[3]; - out0[1] = in1[0]; - out0[2] = in1[1]; - out0[3] = in1[2]; - out1[0] = in1[3]; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 13: out0[0] = amd_bytealign_S (in1[0], in0[3], 1); - out0[1] = amd_bytealign_S (in1[1], in1[0], 1); - out0[2] = amd_bytealign_S (in1[2], in1[1], 1); - out0[3] = amd_bytealign_S (in1[3], in1[2], 1); - out1[0] = amd_bytealign_S ( 0, in1[3], 1); - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 14: out0[0] = amd_bytealign_S (in1[0], in0[3], 2); - out0[1] = amd_bytealign_S (in1[1], in1[0], 2); - out0[2] = amd_bytealign_S (in1[2], in1[1], 2); - out0[3] = amd_bytealign_S (in1[3], in1[2], 2); - out1[0] = amd_bytealign_S ( 0, in1[3], 2); - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 15: out0[0] = amd_bytealign_S (in1[0], in0[3], 3); - out0[1] = amd_bytealign_S (in1[1], in1[0], 3); - out0[2] = amd_bytealign_S (in1[2], in1[1], 3); - out0[3] = amd_bytealign_S (in1[3], in1[2], 3); - out1[0] = amd_bytealign_S ( 0, in1[3], 3); - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 16: out0[0] = in1[0]; - out0[1] = in1[1]; - out0[2] = in1[2]; - out0[3] = in1[3]; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 17: out0[0] = amd_bytealign_S (in1[1], in1[0], 1); - out0[1] = amd_bytealign_S (in1[2], in1[1], 1); - out0[2] = amd_bytealign_S (in1[3], in1[2], 1); - out0[3] = amd_bytealign_S ( 0, in1[3], 1); - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 18: out0[0] = amd_bytealign_S (in1[1], in1[0], 2); - out0[1] = amd_bytealign_S (in1[2], in1[1], 2); - out0[2] = amd_bytealign_S (in1[3], in1[2], 2); - out0[3] = amd_bytealign_S ( 0, in1[3], 2); - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 19: out0[0] = amd_bytealign_S (in1[1], in1[0], 3); - out0[1] = amd_bytealign_S (in1[2], in1[1], 3); - out0[2] = amd_bytealign_S (in1[3], in1[2], 3); - out0[3] = amd_bytealign_S ( 0, in1[3], 3); - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 20: out0[0] = in1[1]; - out0[1] = in1[2]; - out0[2] = in1[3]; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 21: out0[0] = amd_bytealign_S (in1[2], in1[1], 1); - out0[1] = amd_bytealign_S (in1[3], in1[2], 1); - out0[2] = amd_bytealign_S ( 0, in1[3], 1); - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 22: out0[0] = amd_bytealign_S (in1[2], in1[1], 2); - out0[1] = amd_bytealign_S (in1[3], in1[2], 2); - out0[2] = amd_bytealign_S ( 0, in1[3], 2); - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 23: out0[0] = amd_bytealign_S (in1[2], in1[1], 3); - out0[1] = amd_bytealign_S (in1[3], in1[2], 3); - out0[2] = amd_bytealign_S ( 0, in1[3], 3); - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 24: out0[0] = in1[2]; - out0[1] = in1[3]; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 25: out0[0] = amd_bytealign_S (in1[3], in1[2], 1); - out0[1] = amd_bytealign_S ( 0, in1[3], 1); - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 26: out0[0] = amd_bytealign_S (in1[3], in1[2], 2); - out0[1] = amd_bytealign_S ( 0, in1[3], 2); - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 27: out0[0] = amd_bytealign_S (in1[3], in1[2], 3); - out0[1] = amd_bytealign_S ( 0, in1[3], 3); - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 28: out0[0] = in1[3]; - out0[1] = 0; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 29: out0[0] = amd_bytealign_S ( 0, in1[3], 1); - out0[1] = 0; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 30: out0[0] = amd_bytealign_S ( 0, in1[3], 2); - out0[1] = 0; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - case 31: out0[0] = amd_bytealign_S ( 0, in1[3], 3); - out0[1] = 0; - out0[2] = 0; - out0[3] = 0; - out1[0] = 0; - out1[1] = 0; - out1[2] = 0; - out1[3] = 0; - break; - } - #endif -} - -inline void rshift_block_N (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4], const u32 num) -{ - #ifdef IS_NV - switch (num) - { - case 0: out1[3] = in1[3]; - out1[2] = in1[2]; - out1[1] = in1[1]; - out1[0] = in1[0]; - out0[3] = in0[3]; - out0[2] = in0[2]; - out0[1] = in0[1]; - out0[0] = in0[0]; - break; - case 1: out1[3] = __byte_perm_S (in1[2], in1[3], 0x6543); - out1[2] = __byte_perm_S (in1[1], in1[2], 0x6543); - out1[1] = __byte_perm_S (in1[0], in1[1], 0x6543); - out1[0] = __byte_perm_S (in0[3], in1[0], 0x6543); - out0[3] = __byte_perm_S (in0[2], in0[3], 0x6543); - out0[2] = __byte_perm_S (in0[1], in0[2], 0x6543); - out0[1] = __byte_perm_S (in0[0], in0[1], 0x6543); - out0[0] = __byte_perm_S ( 0, in0[0], 0x6543); - break; - case 2: out1[3] = __byte_perm_S (in1[2], in1[3], 0x5432); - out1[2] = __byte_perm_S (in1[1], in1[2], 0x5432); - out1[1] = __byte_perm_S (in1[0], in1[1], 0x5432); - out1[0] = __byte_perm_S (in0[3], in1[0], 0x5432); - out0[3] = __byte_perm_S (in0[2], in0[3], 0x5432); - out0[2] = __byte_perm_S (in0[1], in0[2], 0x5432); - out0[1] = __byte_perm_S (in0[0], in0[1], 0x5432); - out0[0] = __byte_perm_S ( 0, in0[0], 0x5432); - break; - case 3: out1[3] = __byte_perm_S (in1[2], in1[3], 0x4321); - out1[2] = __byte_perm_S (in1[1], in1[2], 0x4321); - out1[1] = __byte_perm_S (in1[0], in1[1], 0x4321); - out1[0] = __byte_perm_S (in0[3], in1[0], 0x4321); - out0[3] = __byte_perm_S (in0[2], in0[3], 0x4321); - out0[2] = __byte_perm_S (in0[1], in0[2], 0x4321); - out0[1] = __byte_perm_S (in0[0], in0[1], 0x4321); - out0[0] = __byte_perm_S ( 0, in0[0], 0x4321); - break; - case 4: out1[3] = in1[2]; - out1[2] = in1[1]; - out1[1] = in1[0]; - out1[0] = in0[3]; - out0[3] = in0[2]; - out0[2] = in0[1]; - out0[1] = in0[0]; - out0[0] = 0; - break; - case 5: out1[3] = __byte_perm_S (in1[1], in1[2], 0x6543); - out1[2] = __byte_perm_S (in1[0], in1[1], 0x6543); - out1[1] = __byte_perm_S (in0[3], in1[0], 0x6543); - out1[0] = __byte_perm_S (in0[2], in0[3], 0x6543); - out0[3] = __byte_perm_S (in0[1], in0[2], 0x6543); - out0[2] = __byte_perm_S (in0[0], in0[1], 0x6543); - out0[1] = __byte_perm_S ( 0, in0[0], 0x6543); - out0[0] = 0; - break; - case 6: out1[3] = __byte_perm_S (in1[1], in1[2], 0x5432); - out1[2] = __byte_perm_S (in1[0], in1[1], 0x5432); - out1[1] = __byte_perm_S (in0[3], in1[0], 0x5432); - out1[0] = __byte_perm_S (in0[2], in0[3], 0x5432); - out0[3] = __byte_perm_S (in0[1], in0[2], 0x5432); - out0[2] = __byte_perm_S (in0[0], in0[1], 0x5432); - out0[1] = __byte_perm_S ( 0, in0[0], 0x5432); - out0[0] = 0; - break; - case 7: out1[3] = __byte_perm_S (in1[1], in1[2], 0x4321); - out1[2] = __byte_perm_S (in1[0], in1[1], 0x4321); - out1[1] = __byte_perm_S (in0[3], in1[0], 0x4321); - out1[0] = __byte_perm_S (in0[2], in0[3], 0x4321); - out0[3] = __byte_perm_S (in0[1], in0[2], 0x4321); - out0[2] = __byte_perm_S (in0[0], in0[1], 0x4321); - out0[1] = __byte_perm_S ( 0, in0[0], 0x4321); - out0[0] = 0; - break; - case 8: out1[3] = in1[1]; - out1[2] = in1[0]; - out1[1] = in0[3]; - out1[0] = in0[2]; - out0[3] = in0[1]; - out0[2] = in0[0]; - out0[1] = 0; - out0[0] = 0; - break; - case 9: out1[3] = __byte_perm_S (in1[0], in1[1], 0x6543); - out1[2] = __byte_perm_S (in0[3], in1[0], 0x6543); - out1[1] = __byte_perm_S (in0[2], in0[3], 0x6543); - out1[0] = __byte_perm_S (in0[1], in0[2], 0x6543); - out0[3] = __byte_perm_S (in0[0], in0[1], 0x6543); - out0[2] = __byte_perm_S ( 0, in0[0], 0x6543); - out0[1] = 0; - out0[0] = 0; - break; - case 10: out1[3] = __byte_perm_S (in1[0], in1[1], 0x5432); - out1[2] = __byte_perm_S (in0[3], in1[0], 0x5432); - out1[1] = __byte_perm_S (in0[2], in0[3], 0x5432); - out1[0] = __byte_perm_S (in0[1], in0[2], 0x5432); - out0[3] = __byte_perm_S (in0[0], in0[1], 0x5432); - out0[2] = __byte_perm_S ( 0, in0[0], 0x5432); - out0[1] = 0; - out0[0] = 0; - break; - case 11: out1[3] = __byte_perm_S (in1[0], in1[1], 0x4321); - out1[2] = __byte_perm_S (in0[3], in1[0], 0x4321); - out1[1] = __byte_perm_S (in0[2], in0[3], 0x4321); - out1[0] = __byte_perm_S (in0[1], in0[2], 0x4321); - out0[3] = __byte_perm_S (in0[0], in0[1], 0x4321); - out0[2] = __byte_perm_S ( 0, in0[0], 0x4321); - out0[1] = 0; - out0[0] = 0; - break; - case 12: out1[3] = in1[0]; - out1[2] = in0[3]; - out1[1] = in0[2]; - out1[0] = in0[1]; - out0[3] = in0[0]; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 13: out1[3] = __byte_perm_S (in0[3], in1[0], 0x6543); - out1[2] = __byte_perm_S (in0[2], in0[3], 0x6543); - out1[1] = __byte_perm_S (in0[1], in0[2], 0x6543); - out1[0] = __byte_perm_S (in0[0], in0[1], 0x6543); - out0[3] = __byte_perm_S ( 0, in0[0], 0x6543); - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 14: out1[3] = __byte_perm_S (in0[3], in1[0], 0x5432); - out1[2] = __byte_perm_S (in0[2], in0[3], 0x5432); - out1[1] = __byte_perm_S (in0[1], in0[2], 0x5432); - out1[0] = __byte_perm_S (in0[0], in0[1], 0x5432); - out0[3] = __byte_perm_S ( 0, in0[0], 0x5432); - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 15: out1[3] = __byte_perm_S (in0[3], in1[0], 0x4321); - out1[2] = __byte_perm_S (in0[2], in0[3], 0x4321); - out1[1] = __byte_perm_S (in0[1], in0[2], 0x4321); - out1[0] = __byte_perm_S (in0[0], in0[1], 0x4321); - out0[3] = __byte_perm_S ( 0, in0[0], 0x4321); - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 16: out1[3] = in0[3]; - out1[2] = in0[2]; - out1[1] = in0[1]; - out1[0] = in0[0]; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 17: out1[3] = __byte_perm_S (in0[2], in0[3], 0x6543); - out1[2] = __byte_perm_S (in0[1], in0[2], 0x6543); - out1[1] = __byte_perm_S (in0[0], in0[1], 0x6543); - out1[0] = __byte_perm_S ( 0, in0[0], 0x6543); - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 18: out1[3] = __byte_perm_S (in0[2], in0[3], 0x5432); - out1[2] = __byte_perm_S (in0[1], in0[2], 0x5432); - out1[1] = __byte_perm_S (in0[0], in0[1], 0x5432); - out1[0] = __byte_perm_S ( 0, in0[0], 0x5432); - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 19: out1[3] = __byte_perm_S (in0[2], in0[3], 0x4321); - out1[2] = __byte_perm_S (in0[1], in0[2], 0x4321); - out1[1] = __byte_perm_S (in0[0], in0[1], 0x4321); - out1[0] = __byte_perm_S ( 0, in0[0], 0x4321); - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 20: out1[3] = in0[2]; - out1[2] = in0[1]; - out1[1] = in0[0]; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 21: out1[3] = __byte_perm_S (in0[1], in0[2], 0x6543); - out1[2] = __byte_perm_S (in0[0], in0[1], 0x6543); - out1[1] = __byte_perm_S ( 0, in0[0], 0x6543); - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 22: out1[3] = __byte_perm_S (in0[1], in0[2], 0x5432); - out1[2] = __byte_perm_S (in0[0], in0[1], 0x5432); - out1[1] = __byte_perm_S ( 0, in0[0], 0x5432); - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 23: out1[3] = __byte_perm_S (in0[1], in0[2], 0x4321); - out1[2] = __byte_perm_S (in0[0], in0[1], 0x4321); - out1[1] = __byte_perm_S ( 0, in0[0], 0x4321); - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 24: out1[3] = in0[1]; - out1[2] = in0[0]; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 25: out1[3] = __byte_perm_S (in0[0], in0[1], 0x6543); - out1[2] = __byte_perm_S ( 0, in0[0], 0x6543); - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 26: out1[3] = __byte_perm_S (in0[0], in0[1], 0x5432); - out1[2] = __byte_perm_S ( 0, in0[0], 0x5432); - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 27: out1[3] = __byte_perm_S (in0[0], in0[1], 0x4321); - out1[2] = __byte_perm_S ( 0, in0[0], 0x4321); - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 28: out1[3] = in0[0]; - out1[2] = 0; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 29: out1[3] = __byte_perm_S ( 0, in0[0], 0x6543); - out1[2] = 0; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 30: out1[3] = __byte_perm_S ( 0, in0[0], 0x5432); - out1[2] = 0; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 31: out1[3] = __byte_perm_S ( 0, in0[0], 0x4321); - out1[2] = 0; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - } - #endif - - #if defined IS_AMD || defined IS_GENERIC - switch (num) - { - case 0: out1[3] = in1[3]; - out1[2] = in1[2]; - out1[1] = in1[1]; - out1[0] = in1[0]; - out0[3] = in0[3]; - out0[2] = in0[2]; - out0[1] = in0[1]; - out0[0] = in0[0]; - break; - case 1: out1[3] = amd_bytealign_S (in1[3], in1[2], 3); - out1[2] = amd_bytealign_S (in1[2], in1[1], 3); - out1[1] = amd_bytealign_S (in1[1], in1[0], 3); - out1[0] = amd_bytealign_S (in1[0], in0[3], 3); - out0[3] = amd_bytealign_S (in0[3], in0[2], 3); - out0[2] = amd_bytealign_S (in0[2], in0[1], 3); - out0[1] = amd_bytealign_S (in0[1], in0[0], 3); - out0[0] = amd_bytealign_S (in0[0], 0, 3); - break; - case 2: out1[3] = amd_bytealign_S (in1[3], in1[2], 2); - out1[2] = amd_bytealign_S (in1[2], in1[1], 2); - out1[1] = amd_bytealign_S (in1[1], in1[0], 2); - out1[0] = amd_bytealign_S (in1[0], in0[3], 2); - out0[3] = amd_bytealign_S (in0[3], in0[2], 2); - out0[2] = amd_bytealign_S (in0[2], in0[1], 2); - out0[1] = amd_bytealign_S (in0[1], in0[0], 2); - out0[0] = amd_bytealign_S (in0[0], 0, 2); - break; - case 3: out1[3] = amd_bytealign_S (in1[3], in1[2], 1); - out1[2] = amd_bytealign_S (in1[2], in1[1], 1); - out1[1] = amd_bytealign_S (in1[1], in1[0], 1); - out1[0] = amd_bytealign_S (in1[0], in0[3], 1); - out0[3] = amd_bytealign_S (in0[3], in0[2], 1); - out0[2] = amd_bytealign_S (in0[2], in0[1], 1); - out0[1] = amd_bytealign_S (in0[1], in0[0], 1); - out0[0] = amd_bytealign_S (in0[0], 0, 1); - break; - case 4: out1[3] = in1[2]; - out1[2] = in1[1]; - out1[1] = in1[0]; - out1[0] = in0[3]; - out0[3] = in0[2]; - out0[2] = in0[1]; - out0[1] = in0[0]; - out0[0] = 0; - break; - case 5: out1[3] = amd_bytealign_S (in1[2], in1[1], 3); - out1[2] = amd_bytealign_S (in1[1], in1[0], 3); - out1[1] = amd_bytealign_S (in1[0], in0[3], 3); - out1[0] = amd_bytealign_S (in0[3], in0[2], 3); - out0[3] = amd_bytealign_S (in0[2], in0[1], 3); - out0[2] = amd_bytealign_S (in0[1], in0[0], 3); - out0[1] = amd_bytealign_S (in0[0], 0, 3); - out0[0] = 0; - break; - case 6: out1[3] = amd_bytealign_S (in1[2], in1[1], 2); - out1[2] = amd_bytealign_S (in1[1], in1[0], 2); - out1[1] = amd_bytealign_S (in1[0], in0[3], 2); - out1[0] = amd_bytealign_S (in0[3], in0[2], 2); - out0[3] = amd_bytealign_S (in0[2], in0[1], 2); - out0[2] = amd_bytealign_S (in0[1], in0[0], 2); - out0[1] = amd_bytealign_S (in0[0], 0, 2); - out0[0] = 0; - break; - case 7: out1[3] = amd_bytealign_S (in1[2], in1[1], 1); - out1[2] = amd_bytealign_S (in1[1], in1[0], 1); - out1[1] = amd_bytealign_S (in1[0], in0[3], 1); - out1[0] = amd_bytealign_S (in0[3], in0[2], 1); - out0[3] = amd_bytealign_S (in0[2], in0[1], 1); - out0[2] = amd_bytealign_S (in0[1], in0[0], 1); - out0[1] = amd_bytealign_S (in0[0], 0, 1); - out0[0] = 0; - break; - case 8: out1[3] = in1[1]; - out1[2] = in1[0]; - out1[1] = in0[3]; - out1[0] = in0[2]; - out0[3] = in0[1]; - out0[2] = in0[0]; - out0[1] = 0; - out0[0] = 0; - break; - case 9: out1[3] = amd_bytealign_S (in1[1], in1[0], 3); - out1[2] = amd_bytealign_S (in1[0], in0[3], 3); - out1[1] = amd_bytealign_S (in0[3], in0[2], 3); - out1[0] = amd_bytealign_S (in0[2], in0[1], 3); - out0[3] = amd_bytealign_S (in0[1], in0[0], 3); - out0[2] = amd_bytealign_S (in0[0], 0, 3); - out0[1] = 0; - out0[0] = 0; - break; - case 10: out1[3] = amd_bytealign_S (in1[1], in1[0], 2); - out1[2] = amd_bytealign_S (in1[0], in0[3], 2); - out1[1] = amd_bytealign_S (in0[3], in0[2], 2); - out1[0] = amd_bytealign_S (in0[2], in0[1], 2); - out0[3] = amd_bytealign_S (in0[1], in0[0], 2); - out0[2] = amd_bytealign_S (in0[0], 0, 2); - out0[1] = 0; - out0[0] = 0; - break; - case 11: out1[3] = amd_bytealign_S (in1[1], in1[0], 1); - out1[2] = amd_bytealign_S (in1[0], in0[3], 1); - out1[1] = amd_bytealign_S (in0[3], in0[2], 1); - out1[0] = amd_bytealign_S (in0[2], in0[1], 1); - out0[3] = amd_bytealign_S (in0[1], in0[0], 1); - out0[2] = amd_bytealign_S (in0[0], 0, 1); - out0[1] = 0; - out0[0] = 0; - break; - case 12: out1[3] = in1[0]; - out1[2] = in0[3]; - out1[1] = in0[2]; - out1[0] = in0[1]; - out0[3] = in0[0]; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 13: out1[3] = amd_bytealign_S (in1[0], in0[3], 3); - out1[2] = amd_bytealign_S (in0[3], in0[2], 3); - out1[1] = amd_bytealign_S (in0[2], in0[1], 3); - out1[0] = amd_bytealign_S (in0[1], in0[0], 3); - out0[3] = amd_bytealign_S (in0[0], 0, 3); - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 14: out1[3] = amd_bytealign_S (in1[0], in0[3], 2); - out1[2] = amd_bytealign_S (in0[3], in0[2], 2); - out1[1] = amd_bytealign_S (in0[2], in0[1], 2); - out1[0] = amd_bytealign_S (in0[1], in0[0], 2); - out0[3] = amd_bytealign_S (in0[0], 0, 2); - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 15: out1[3] = amd_bytealign_S (in1[0], in0[3], 1); - out1[2] = amd_bytealign_S (in0[3], in0[2], 1); - out1[1] = amd_bytealign_S (in0[2], in0[1], 1); - out1[0] = amd_bytealign_S (in0[1], in0[0], 1); - out0[3] = amd_bytealign_S (in0[0], 0, 1); - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 16: out1[3] = in0[3]; - out1[2] = in0[2]; - out1[1] = in0[1]; - out1[0] = in0[0]; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 17: out1[3] = amd_bytealign_S (in0[3], in0[2], 3); - out1[2] = amd_bytealign_S (in0[2], in0[1], 3); - out1[1] = amd_bytealign_S (in0[1], in0[0], 3); - out1[0] = amd_bytealign_S (in0[0], 0, 3); - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 18: out1[3] = amd_bytealign_S (in0[3], in0[2], 2); - out1[2] = amd_bytealign_S (in0[2], in0[1], 2); - out1[1] = amd_bytealign_S (in0[1], in0[0], 2); - out1[0] = amd_bytealign_S (in0[0], 0, 2); - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 19: out1[3] = amd_bytealign_S (in0[3], in0[2], 1); - out1[2] = amd_bytealign_S (in0[2], in0[1], 1); - out1[1] = amd_bytealign_S (in0[1], in0[0], 1); - out1[0] = amd_bytealign_S (in0[0], 0, 1); - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 20: out1[3] = in0[2]; - out1[2] = in0[1]; - out1[1] = in0[0]; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 21: out1[3] = amd_bytealign_S (in0[2], in0[1], 3); - out1[2] = amd_bytealign_S (in0[1], in0[0], 3); - out1[1] = amd_bytealign_S (in0[0], 0, 3); - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 22: out1[3] = amd_bytealign_S (in0[2], in0[1], 2); - out1[2] = amd_bytealign_S (in0[1], in0[0], 2); - out1[1] = amd_bytealign_S (in0[0], 0, 2); - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 23: out1[3] = amd_bytealign_S (in0[2], in0[1], 1); - out1[2] = amd_bytealign_S (in0[1], in0[0], 1); - out1[1] = amd_bytealign_S (in0[0], 0, 1); - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 24: out1[3] = in0[1]; - out1[2] = in0[0]; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 25: out1[3] = amd_bytealign_S (in0[1], in0[0], 3); - out1[2] = amd_bytealign_S (in0[0], 0, 3); - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 26: out1[3] = amd_bytealign_S (in0[1], in0[0], 2); - out1[2] = amd_bytealign_S (in0[0], 0, 2); - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 27: out1[3] = amd_bytealign_S (in0[1], in0[0], 1); - out1[2] = amd_bytealign_S (in0[0], 0, 1); - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 28: out1[3] = in0[0]; - out1[2] = 0; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 29: out1[3] = amd_bytealign_S (in0[0], 0, 3); - out1[2] = 0; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 30: out1[3] = amd_bytealign_S (in0[0], 0, 2); - out1[2] = 0; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - case 31: out1[3] = amd_bytealign_S (in0[0], 0, 1); - out1[2] = 0; - out1[1] = 0; - out1[0] = 0; - out0[3] = 0; - out0[2] = 0; - out0[1] = 0; - out0[0] = 0; - break; - } - #endif -} - -inline void append_block1 (const u32 offset, u32 dst0[4], u32 dst1[4], const u32 src_r0) -{ - u32 tmp[2]; - - switch (offset & 3) - { - case 0: tmp[0] = src_r0; - tmp[1] = 0; - break; - case 1: tmp[0] = src_r0 << 8; - tmp[1] = src_r0 >> 24; - break; - case 2: tmp[0] = src_r0 << 16; - tmp[1] = src_r0 >> 16; - break; - case 3: tmp[0] = src_r0 << 24; - tmp[1] = src_r0 >> 8; - break; - } - - switch (offset / 4) - { - case 0: dst0[0] |= tmp[0]; - dst0[1] = tmp[1]; - break; - case 1: dst0[1] |= tmp[0]; - dst0[2] = tmp[1]; - break; - case 2: dst0[2] |= tmp[0]; - dst0[3] = tmp[1]; - break; - case 3: dst0[3] |= tmp[0]; - dst1[0] = tmp[1]; - break; - case 4: dst1[0] |= tmp[0]; - dst1[1] = tmp[1]; - break; - case 5: dst1[1] |= tmp[0]; - dst1[2] = tmp[1]; - break; - case 6: dst1[2] |= tmp[0]; - dst1[3] = tmp[1]; - break; - case 7: dst1[3] |= tmp[0]; - break; - } -} - -inline void append_block8 (const u32 offset, u32 dst0[4], u32 dst1[4], const u32 src_l0[4], const u32 src_l1[4], const u32 src_r0[4], const u32 src_r1[4]) -{ -/* - #ifdef IS_NV - switch (offset) - { - case 0: - dst0[0] = src_r0[0]; - dst0[1] = src_r0[1]; - dst0[2] = src_r0[2]; - dst0[3] = src_r0[3]; - dst1[0] = src_r1[0]; - dst1[1] = src_r1[1]; - dst1[2] = src_r1[2]; - dst1[3] = src_r1[3]; - break; - - case 1: - dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x6540); - dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); - dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); - dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); - dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543); - dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543); - dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x6543); - dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x6543); - break; - - case 2: - dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x5410); - dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); - dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); - dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); - dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432); - dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432); - dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x5432); - dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x5432); - break; - - case 3: - dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x4210); - dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); - dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); - dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); - dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321); - dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321); - dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x4321); - dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x4321); - break; - - case 4: - dst0[1] = src_r0[0]; - dst0[2] = src_r0[1]; - dst0[3] = src_r0[2]; - dst1[0] = src_r0[3]; - dst1[1] = src_r1[0]; - dst1[2] = src_r1[1]; - dst1[3] = src_r1[2]; - break; - - case 5: - dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x6540); - dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); - dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); - dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); - dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543); - dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543); - dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x6543); - break; - - case 6: - dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x5410); - dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); - dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); - dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); - dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432); - dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432); - dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x5432); - break; - - case 7: - dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x4210); - dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); - dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); - dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); - dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321); - dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321); - dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x4321); - break; - - case 8: - dst0[2] = src_r0[0]; - dst0[3] = src_r0[1]; - dst1[0] = src_r0[2]; - dst1[1] = src_r0[3]; - dst1[2] = src_r1[0]; - dst1[3] = src_r1[1]; - break; - - case 9: - dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x6540); - dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); - dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); - dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); - dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543); - dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543); - break; - - case 10: - dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x5410); - dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); - dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); - dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); - dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432); - dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432); - break; - - case 11: - dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x4210); - dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); - dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); - dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); - dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321); - dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321); - break; - - case 12: - dst0[3] = src_r0[0]; - dst1[0] = src_r0[1]; - dst1[1] = src_r0[2]; - dst1[2] = src_r0[3]; - dst1[3] = src_r1[0]; - break; - - case 13: - dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x6540); - dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); - dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); - dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); - dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543); - break; - - case 14: - dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x5410); - dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); - dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); - dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); - dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432); - break; - - case 15: - dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x4210); - dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); - dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); - dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); - dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321); - break; - - case 16: - dst1[0] = src_r0[0]; - dst1[1] = src_r0[1]; - dst1[2] = src_r0[2]; - dst1[3] = src_r0[3]; - break; - - case 17: - dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x6540); - dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); - dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); - dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543); - break; - - case 18: - dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x5410); - dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); - dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); - dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432); - break; - - case 19: - dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x4210); - dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); - dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); - dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321); - break; - - case 20: - dst1[1] = src_r0[0]; - dst1[2] = src_r0[1]; - dst1[3] = src_r0[2]; - break; - - case 21: - dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x6540); - dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); - dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543); - break; - - case 22: - dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x5410); - dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); - dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432); - break; - - case 23: - dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x4210); - dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); - dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321); - break; - - case 24: - dst1[2] = src_r0[0]; - dst1[3] = src_r0[1]; - break; - - case 25: - dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x6540); - dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543); - break; - - case 26: - dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x5410); - dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432); - break; - - case 27: - dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x4210); - dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321); - break; - - case 28: - dst1[3] = src_r0[0]; - break; - - case 29: - dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x6540); - break; - - case 30: - dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x5410); - break; - - case 31: - dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x4210); - break; - } - #endif - - #if defined IS_AMD || defined IS_GENERIC -*/ - switch (offset) - { - case 31: - dst1[3] = src_l1[3] | src_r0[0] << 24; - break; - case 30: - dst1[3] = src_l1[3] | src_r0[0] << 16; - break; - case 29: - dst1[3] = src_l1[3] | src_r0[0] << 8; - break; - case 28: - dst1[3] = src_r0[0]; - break; - case 27: - dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 1); - dst1[2] = src_l1[2] | src_r0[0] << 24; - break; - case 26: - dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 2); - dst1[2] = src_l1[2] | src_r0[0] << 16; - break; - case 25: - dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 3); - dst1[2] = src_l1[2] | src_r0[0] << 8; - break; - case 24: - dst1[3] = src_r0[1]; - dst1[2] = src_r0[0]; - break; - case 23: - dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 1); - dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 1); - dst1[1] = src_l1[1] | src_r0[0] << 24; - break; - case 22: - dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 2); - dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 2); - dst1[1] = src_l1[1] | src_r0[0] << 16; - break; - case 21: - dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 3); - dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 3); - dst1[1] = src_l1[1] | src_r0[0] << 8; - break; - case 20: - dst1[3] = src_r0[2]; - dst1[2] = src_r0[1]; - dst1[1] = src_r0[0]; - break; - case 19: - dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 1); - dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 1); - dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 1); - dst1[0] = src_l1[0] | src_r0[0] << 24; - break; - case 18: - dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 2); - dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 2); - dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 2); - dst1[0] = src_l1[0] | src_r0[0] << 16; - break; - case 17: - dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 3); - dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 3); - dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 3); - dst1[0] = src_l1[0] | src_r0[0] << 8; - break; - case 16: - dst1[3] = src_r0[3]; - dst1[2] = src_r0[2]; - dst1[1] = src_r0[1]; - dst1[0] = src_r0[0]; - break; - case 15: - dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 1); - dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 1); - dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 1); - dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 1); - dst0[3] = src_l0[3] | src_r0[0] << 24; - break; - case 14: - dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 2); - dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 2); - dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 2); - dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 2); - dst0[3] = src_l0[3] | src_r0[0] << 16; - break; - case 13: - dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 3); - dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 3); - dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 3); - dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 3); - dst0[3] = src_l0[3] | src_r0[0] << 8; - break; - case 12: - dst1[3] = src_r1[0]; - dst1[2] = src_r0[3]; - dst1[1] = src_r0[2]; - dst1[0] = src_r0[1]; - dst0[3] = src_r0[0]; - break; - case 11: - dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 1); - dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 1); - dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 1); - dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 1); - dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 1); - dst0[2] = src_l0[2] | src_r0[0] << 24; - break; - case 10: - dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 2); - dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 2); - dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 2); - dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 2); - dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 2); - dst0[2] = src_l0[2] | src_r0[0] << 16; - break; - case 9: - dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 3); - dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 3); - dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 3); - dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 3); - dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 3); - dst0[2] = src_l0[2] | src_r0[0] << 8; - break; - case 8: - dst1[3] = src_r1[1]; - dst1[2] = src_r1[0]; - dst1[1] = src_r0[3]; - dst1[0] = src_r0[2]; - dst0[3] = src_r0[1]; - dst0[2] = src_r0[0]; - break; - case 7: - dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 1); - dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 1); - dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 1); - dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 1); - dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 1); - dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 1); - dst0[1] = src_l0[1] | src_r0[0] << 24; - break; - case 6: - dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 2); - dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 2); - dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 2); - dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 2); - dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 2); - dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 2); - dst0[1] = src_l0[1] | src_r0[0] << 16; - break; - case 5: - dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 3); - dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 3); - dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 3); - dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 3); - dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 3); - dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 3); - dst0[1] = src_l0[1] | src_r0[0] << 8; - break; - case 4: - dst1[3] = src_r1[2]; - dst1[2] = src_r1[1]; - dst1[1] = src_r1[0]; - dst1[0] = src_r0[3]; - dst0[3] = src_r0[2]; - dst0[2] = src_r0[1]; - dst0[1] = src_r0[0]; - break; - case 3: - dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 1); - dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 1); - dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 1); - dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 1); - dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 1); - dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 1); - dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 1); - dst0[0] = src_l0[0] | src_r0[0] << 24; - break; - case 2: - dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 2); - dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 2); - dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 2); - dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 2); - dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 2); - dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 2); - dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 2); - dst0[0] = src_l0[0] | src_r0[0] << 16; - break; - case 1: - dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 3); - dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 3); - dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 3); - dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 3); - dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 3); - dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 3); - dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 3); - dst0[0] = src_l0[0] | src_r0[0] << 8; - break; - case 0: - dst1[3] = src_r1[3]; - dst1[2] = src_r1[2]; - dst1[1] = src_r1[1]; - dst1[0] = src_r1[0]; - dst0[3] = src_r0[3]; - dst0[2] = src_r0[2]; - dst0[1] = src_r0[1]; - dst0[0] = src_r0[0]; - break; - } -// #endif -} - -inline void reverse_block (u32 in0[4], u32 in1[4], u32 out0[4], u32 out1[4], const u32 len) -{ - rshift_block_N (in0, in1, out0, out1, 32 - len); - - u32 tib40[4]; - u32 tib41[4]; - - tib40[0] = out1[3]; - tib40[1] = out1[2]; - tib40[2] = out1[1]; - tib40[3] = out1[0]; - tib41[0] = out0[3]; - tib41[1] = out0[2]; - tib41[2] = out0[1]; - tib41[3] = out0[0]; - - out0[0] = swap32_S (tib40[0]); - out0[1] = swap32_S (tib40[1]); - out0[2] = swap32_S (tib40[2]); - out0[3] = swap32_S (tib40[3]); - out1[0] = swap32_S (tib41[0]); - out1[1] = swap32_S (tib41[1]); - out1[2] = swap32_S (tib41[2]); - out1[3] = swap32_S (tib41[3]); -} - -inline u32 rule_op_mangle_lrest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - buf0[0] |= (generate_cmask (buf0[0])); - buf0[1] |= (generate_cmask (buf0[1])); - buf0[2] |= (generate_cmask (buf0[2])); - buf0[3] |= (generate_cmask (buf0[3])); - buf1[0] |= (generate_cmask (buf1[0])); - buf1[1] |= (generate_cmask (buf1[1])); - buf1[2] |= (generate_cmask (buf1[2])); - buf1[3] |= (generate_cmask (buf1[3])); - - return in_len; -} - -inline u32 rule_op_mangle_urest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - buf0[0] &= ~(generate_cmask (buf0[0])); - buf0[1] &= ~(generate_cmask (buf0[1])); - buf0[2] &= ~(generate_cmask (buf0[2])); - buf0[3] &= ~(generate_cmask (buf0[3])); - buf1[0] &= ~(generate_cmask (buf1[0])); - buf1[1] &= ~(generate_cmask (buf1[1])); - buf1[2] &= ~(generate_cmask (buf1[2])); - buf1[3] &= ~(generate_cmask (buf1[3])); - - return in_len; -} - -inline u32 rule_op_mangle_lrest_ufirst (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - rule_op_mangle_lrest (p0, p1, buf0, buf1, in_len); - - buf0[0] &= ~(0x00000020 & generate_cmask (buf0[0])); - - return in_len; -} - -inline u32 rule_op_mangle_urest_lfirst (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - rule_op_mangle_urest (p0, p1, buf0, buf1, in_len); - - buf0[0] |= (0x00000020 & generate_cmask (buf0[0])); - - return in_len; -} - -inline u32 rule_op_mangle_trest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - buf0[0] ^= (generate_cmask (buf0[0])); - buf0[1] ^= (generate_cmask (buf0[1])); - buf0[2] ^= (generate_cmask (buf0[2])); - buf0[3] ^= (generate_cmask (buf0[3])); - buf1[0] ^= (generate_cmask (buf1[0])); - buf1[1] ^= (generate_cmask (buf1[1])); - buf1[2] ^= (generate_cmask (buf1[2])); - buf1[3] ^= (generate_cmask (buf1[3])); - - return in_len; -} - -inline u32 rule_op_mangle_toggle_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - const u32 tmp = 0x20u << ((p0 & 3) * 8); - - switch (p0 / 4) - { - case 0: buf0[0] ^= (tmp & generate_cmask (buf0[0])); break; - case 1: buf0[1] ^= (tmp & generate_cmask (buf0[1])); break; - case 2: buf0[2] ^= (tmp & generate_cmask (buf0[2])); break; - case 3: buf0[3] ^= (tmp & generate_cmask (buf0[3])); break; - case 4: buf1[0] ^= (tmp & generate_cmask (buf1[0])); break; - case 5: buf1[1] ^= (tmp & generate_cmask (buf1[1])); break; - case 6: buf1[2] ^= (tmp & generate_cmask (buf1[2])); break; - case 7: buf1[3] ^= (tmp & generate_cmask (buf1[3])); break; - } - - return in_len; -} - -inline u32 rule_op_mangle_reverse (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - reverse_block (buf0, buf1, buf0, buf1, in_len); - - return in_len; -} - -inline u32 rule_op_mangle_dupeword (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if ((in_len + in_len) >= 32) return (in_len); - - u32 out_len = in_len; - - append_block8 (out_len, buf0, buf1, buf0, buf1, buf0, buf1); - - out_len += in_len; - - return out_len; -} - -inline u32 rule_op_mangle_dupeword_times (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (((in_len * p0) + in_len) >= 32) return (in_len); - - u32 out_len = in_len; - - u32 tib40[4]; - u32 tib41[4]; - - tib40[0] = buf0[0]; - tib40[1] = buf0[1]; - tib40[2] = buf0[2]; - tib40[3] = buf0[3]; - tib41[0] = buf1[0]; - tib41[1] = buf1[1]; - tib41[2] = buf1[2]; - tib41[3] = buf1[3]; - - for (u32 i = 0; i < p0; i++) - { - append_block8 (out_len, buf0, buf1, buf0, buf1, tib40, tib41); - - out_len += in_len; - } - - return out_len; -} - -inline u32 rule_op_mangle_reflect (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if ((in_len + in_len) >= 32) return (in_len); - - u32 out_len = in_len; - - u32 tib40[4]; - u32 tib41[4]; - - reverse_block (buf0, buf1, tib40, tib41, out_len); - - append_block8 (out_len, buf0, buf1, buf0, buf1, tib40, tib41); - - out_len += in_len; - - return out_len; -} - -inline u32 rule_op_mangle_append (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if ((in_len + 1) >= 32) return (in_len); - - u32 out_len = in_len; - - append_block1 (out_len, buf0, buf1, p0); - - out_len++; - - return out_len; -} - -inline u32 rule_op_mangle_prepend (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if ((in_len + 1) >= 32) return (in_len); - - u32 out_len = in_len; - - rshift_block (buf0, buf1, buf0, buf1); - - buf0[0] = buf0[0] | p0; - - out_len++; - - return out_len; -} - -inline u32 rule_op_mangle_rotate_left (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (in_len == 0) return (in_len); - - const u32 in_len1 = in_len - 1; - - const u32 sh = (in_len1 & 3) * 8; - - const u32 tmp = (buf0[0] & 0xff) << sh; - - lshift_block (buf0, buf1, buf0, buf1); - - switch (in_len1 / 4) - { - case 0: buf0[0] |= tmp; break; - case 1: buf0[1] |= tmp; break; - case 2: buf0[2] |= tmp; break; - case 3: buf0[3] |= tmp; break; - case 4: buf1[0] |= tmp; break; - case 5: buf1[1] |= tmp; break; - case 6: buf1[2] |= tmp; break; - case 7: buf1[3] |= tmp; break; - } - - return in_len; -} - -inline u32 rule_op_mangle_rotate_right (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (in_len == 0) return (in_len); - - const u32 in_len1 = in_len - 1; - - const u32 sh = (in_len1 & 3) * 8; - - u32 tmp = 0; - - switch (in_len1 / 4) - { - case 0: tmp = (buf0[0] >> sh) & 0xff; break; - case 1: tmp = (buf0[1] >> sh) & 0xff; break; - case 2: tmp = (buf0[2] >> sh) & 0xff; break; - case 3: tmp = (buf0[3] >> sh) & 0xff; break; - case 4: tmp = (buf1[0] >> sh) & 0xff; break; - case 5: tmp = (buf1[1] >> sh) & 0xff; break; - case 6: tmp = (buf1[2] >> sh) & 0xff; break; - case 7: tmp = (buf1[3] >> sh) & 0xff; break; - } - - rshift_block (buf0, buf1, buf0, buf1); - - buf0[0] |= tmp; - - truncate_right (buf0, buf1, in_len); - - return in_len; -} - -inline u32 rule_op_mangle_delete_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (in_len == 0) return (in_len); - - const u32 in_len1 = in_len - 1; - - lshift_block (buf0, buf1, buf0, buf1); - - return in_len1; -} - -inline u32 rule_op_mangle_delete_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (in_len == 0) return (in_len); - - const u32 in_len1 = in_len - 1; - - const u32 tmp = (1 << ((in_len1 & 3) * 8)) - 1; - - switch (in_len1 / 4) - { - case 0: buf0[0] &= tmp; break; - case 1: buf0[1] &= tmp; break; - case 2: buf0[2] &= tmp; break; - case 3: buf0[3] &= tmp; break; - case 4: buf1[0] &= tmp; break; - case 5: buf1[1] &= tmp; break; - case 6: buf1[2] &= tmp; break; - case 7: buf1[3] &= tmp; break; - } - - return in_len1; -} - -inline u32 rule_op_mangle_delete_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - u32 out_len = in_len; - - u32 tib40[4]; - u32 tib41[4]; - - lshift_block (buf0, buf1, tib40, tib41); - - const u32 ml = (1 << ((p0 & 3) * 8)) - 1; - const u32 mr = ~ml; - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & ml) - | (tib40[0] & mr); - buf0[1] = tib40[1]; - buf0[2] = tib40[2]; - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 1: buf0[1] = (buf0[1] & ml) - | (tib40[1] & mr); - buf0[2] = tib40[2]; - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 2: buf0[2] = (buf0[2] & ml) - | (tib40[2] & mr); - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 3: buf0[3] = (buf0[3] & ml) - | (tib40[3] & mr); - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 4: buf1[0] = (buf1[0] & ml) - | (tib41[0] & mr); - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 5: buf1[1] = (buf1[1] & ml) - | (tib41[1] & mr); - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 6: buf1[2] = (buf1[2] & ml) - | (tib41[2] & mr); - buf1[3] = tib41[3]; - break; - case 7: buf1[3] = (buf1[3] & ml) - | (tib41[3] & mr); - break; - } - - out_len--; - - return out_len; -} - -inline u32 rule_op_mangle_extract (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - if ((p0 + p1) > in_len) return (in_len); - - u32 out_len = p1; - - lshift_block_N (buf0, buf1, buf0, buf1, p0); - - truncate_right (buf0, buf1, out_len); - - return out_len; -} - -inline u32 rule_op_mangle_omit (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - if ((p0 + p1) > in_len) return (in_len); - - u32 out_len = in_len; - - u32 tib40[4]; - u32 tib41[4]; - - tib40[0] = 0; - tib40[1] = 0; - tib40[2] = 0; - tib40[3] = 0; - tib41[0] = 0; - tib41[1] = 0; - tib41[2] = 0; - tib41[3] = 0; - - lshift_block_N (buf0, buf1, tib40, tib41, p1); - - const u32 ml = (1 << ((p0 & 3) * 8)) - 1; - const u32 mr = ~ml; - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & ml) - | (tib40[0] & mr); - buf0[1] = tib40[1]; - buf0[2] = tib40[2]; - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 1: buf0[1] = (buf0[1] & ml) - | (tib40[1] & mr); - buf0[2] = tib40[2]; - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 2: buf0[2] = (buf0[2] & ml) - | (tib40[2] & mr); - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 3: buf0[3] = (buf0[3] & ml) - | (tib40[3] & mr); - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 4: buf1[0] = (buf1[0] & ml) - | (tib41[0] & mr); - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 5: buf1[1] = (buf1[1] & ml) - | (tib41[1] & mr); - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 6: buf1[2] = (buf1[2] & ml) - | (tib41[2] & mr); - buf1[3] = tib41[3]; - break; - case 7: buf1[3] = (buf1[3] & ml) - | (tib41[3] & mr); - break; - } - - out_len -= p1; - - return out_len; -} - -inline u32 rule_op_mangle_insert (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 > in_len) return (in_len); - - if ((in_len + 1) >= 32) return (in_len); - - u32 out_len = in_len; - - u32 tib40[4]; - u32 tib41[4]; - - rshift_block (buf0, buf1, tib40, tib41); - - const u32 p1n = p1 << ((p0 & 3) * 8); - - const u32 ml = (1 << ((p0 & 3) * 8)) - 1; - - const u32 mr = 0xffffff00 << ((p0 & 3) * 8); - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & ml) | p1n | (tib40[0] & mr); - buf0[1] = tib40[1]; - buf0[2] = tib40[2]; - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 1: buf0[1] = (buf0[1] & ml) | p1n | (tib40[1] & mr); - buf0[2] = tib40[2]; - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 2: buf0[2] = (buf0[2] & ml) | p1n | (tib40[2] & mr); - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 3: buf0[3] = (buf0[3] & ml) | p1n | (tib40[3] & mr); - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 4: buf1[0] = (buf1[0] & ml) | p1n | (tib41[0] & mr); - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 5: buf1[1] = (buf1[1] & ml) | p1n | (tib41[1] & mr); - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - break; - case 6: buf1[2] = (buf1[2] & ml) | p1n | (tib41[2] & mr); - buf1[3] = tib41[3]; - break; - case 7: buf1[3] = (buf1[3] & ml) | p1n | (tib41[3] & mr); - break; - } - - out_len++; - - return out_len; -} - -inline u32 rule_op_mangle_overstrike (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - const u32 p1n = p1 << ((p0 & 3) * 8); - - const u32 m = ~(0xffu << ((p0 & 3) * 8)); - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & m) | p1n; break; - case 1: buf0[1] = (buf0[1] & m) | p1n; break; - case 2: buf0[2] = (buf0[2] & m) | p1n; break; - case 3: buf0[3] = (buf0[3] & m) | p1n; break; - case 4: buf1[0] = (buf1[0] & m) | p1n; break; - case 5: buf1[1] = (buf1[1] & m) | p1n; break; - case 6: buf1[2] = (buf1[2] & m) | p1n; break; - case 7: buf1[3] = (buf1[3] & m) | p1n; break; - } - - return in_len; -} - -inline u32 rule_op_mangle_truncate_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - truncate_right (buf0, buf1, p0); - - return p0; -} - -inline u32 rule_op_mangle_replace (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - #ifdef IS_NV - for (u32 i = 0; i < in_len; i++) - { - switch (i) - { - case 0: if ((__byte_perm_S (buf0[0], 0, 0x6540)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7650); - break; - case 1: if ((__byte_perm_S (buf0[0], 0, 0x6541)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7604); - break; - case 2: if ((__byte_perm_S (buf0[0], 0, 0x6542)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7054); - break; - case 3: if ((__byte_perm_S (buf0[0], 0, 0x6543)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x0654); - break; - case 4: if ((__byte_perm_S (buf0[1], 0, 0x6540)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7650); - break; - case 5: if ((__byte_perm_S (buf0[1], 0, 0x6541)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7604); - break; - case 6: if ((__byte_perm_S (buf0[1], 0, 0x6542)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7054); - break; - case 7: if ((__byte_perm_S (buf0[1], 0, 0x6543)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x0654); - break; - case 8: if ((__byte_perm_S (buf0[2], 0, 0x6540)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7650); - break; - case 9: if ((__byte_perm_S (buf0[2], 0, 0x6541)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7604); - break; - case 10: if ((__byte_perm_S (buf0[2], 0, 0x6542)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7054); - break; - case 11: if ((__byte_perm_S (buf0[2], 0, 0x6543)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x0654); - break; - case 12: if ((__byte_perm_S (buf0[3], 0, 0x6540)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7650); - break; - case 13: if ((__byte_perm_S (buf0[3], 0, 0x6541)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7604); - break; - case 14: if ((__byte_perm_S (buf0[3], 0, 0x6542)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7054); - break; - case 15: if ((__byte_perm_S (buf0[3], 0, 0x6543)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x0654); - break; - case 16: if ((__byte_perm_S (buf1[0], 0, 0x6540)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7650); - break; - case 17: if ((__byte_perm_S (buf1[0], 0, 0x6541)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7604); - break; - case 18: if ((__byte_perm_S (buf1[0], 0, 0x6542)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7054); - break; - case 19: if ((__byte_perm_S (buf1[0], 0, 0x6543)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x0654); - break; - case 20: if ((__byte_perm_S (buf1[1], 0, 0x6540)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7650); - break; - case 21: if ((__byte_perm_S (buf1[1], 0, 0x6541)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7604); - break; - case 22: if ((__byte_perm_S (buf1[1], 0, 0x6542)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7054); - break; - case 23: if ((__byte_perm_S (buf1[1], 0, 0x6543)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x0654); - break; - case 24: if ((__byte_perm_S (buf1[2], 0, 0x6540)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7650); - break; - case 25: if ((__byte_perm_S (buf1[2], 0, 0x6541)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7604); - break; - case 26: if ((__byte_perm_S (buf1[2], 0, 0x6542)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7054); - break; - case 27: if ((__byte_perm_S (buf1[2], 0, 0x6543)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x0654); - break; - case 28: if ((__byte_perm_S (buf1[3], 0, 0x6540)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7650); - break; - case 29: if ((__byte_perm_S (buf1[3], 0, 0x6541)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7604); - break; - case 30: if ((__byte_perm_S (buf1[3], 0, 0x6542)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7054); - break; - case 31: if ((__byte_perm_S (buf1[3], 0, 0x6543)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x0654); - break; - } - } - #endif - - #if defined IS_AMD || defined IS_GENERIC - const uchar4 tmp0 = (uchar4) (p0); - const uchar4 tmp1 = (uchar4) (p1); - - uchar4 tmp; - - tmp = as_uchar4 (buf0[0]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[0] = as_uint (tmp); - tmp = as_uchar4 (buf0[1]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[1] = as_uint (tmp); - tmp = as_uchar4 (buf0[2]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[2] = as_uint (tmp); - tmp = as_uchar4 (buf0[3]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[3] = as_uint (tmp); - tmp = as_uchar4 (buf1[0]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[0] = as_uint (tmp); - tmp = as_uchar4 (buf1[1]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[1] = as_uint (tmp); - tmp = as_uchar4 (buf1[2]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[2] = as_uint (tmp); - tmp = as_uchar4 (buf1[3]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[3] = as_uint (tmp); - #endif - - return in_len; -} - -inline u32 rule_op_mangle_purgechar (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - // TODO - return in_len; -} - -inline u32 rule_op_mangle_togglecase_rec (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - // TODO - return in_len; -} - -inline u32 rule_op_mangle_dupechar_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if ( in_len == 0) return (in_len); - if ((in_len + p0) >= 32) return (in_len); - - u32 out_len = in_len; - - const u32 tmp = buf0[0] & 0xFF; - - rshift_block_N (buf0, buf1, buf0, buf1, p0); - - #ifdef IS_NV - switch (p0) - { - case 1: buf0[0] |= tmp; - break; - case 2: buf0[0] |= __byte_perm_S (tmp, 0, 0x5400); - break; - case 3: buf0[0] |= __byte_perm_S (tmp, 0, 0x4000); - break; - case 4: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - break; - case 5: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= tmp; - break; - case 6: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x5400); - break; - case 7: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x4000); - break; - case 8: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - break; - case 9: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= tmp; - break; - case 10: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x5400); - break; - case 11: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x4000); - break; - case 12: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - break; - case 13: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= tmp; - break; - case 14: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x5400); - break; - case 15: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x4000); - break; - case 16: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - break; - case 17: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= tmp; - break; - case 18: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x5400); - break; - case 19: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x4000); - break; - case 20: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - break; - case 21: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= tmp; - break; - case 22: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x5400); - break; - case 23: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x4000); - break; - case 24: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); - break; - case 25: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[2] |= tmp; - break; - case 26: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[2] |= __byte_perm_S (tmp, 0, 0x5400); - break; - case 27: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[2] |= __byte_perm_S (tmp, 0, 0x4000); - break; - case 28: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[2] |= __byte_perm_S (tmp, 0, 0x0000); - break; - case 29: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[3] |= tmp; - break; - case 30: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[3] |= __byte_perm_S (tmp, 0, 0x5400); - break; - case 31: buf0[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf0[3] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[0] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[1] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[2] |= __byte_perm_S (tmp, 0, 0x0000); - buf1[3] |= __byte_perm_S (tmp, 0, 0x4000); - break; - } - #endif - - #if defined IS_AMD || defined IS_GENERIC - switch (p0) - { - case 1: buf0[0] |= tmp << 0; - break; - case 2: buf0[0] |= tmp << 0 | tmp << 8; - break; - case 3: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16; - break; - case 4: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - break; - case 5: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0; - break; - case 6: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8; - break; - case 7: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16; - break; - case 8: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - break; - case 9: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0; - break; - case 10: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8; - break; - case 11: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16; - break; - case 12: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - break; - case 13: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0; - break; - case 14: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8; - break; - case 15: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16; - break; - case 16: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - break; - case 17: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0; - break; - case 18: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8; - break; - case 19: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16; - break; - case 20: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - break; - case 21: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0; - break; - case 22: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8; - break; - case 23: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16; - break; - case 24: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - break; - case 25: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[2] |= tmp << 0; - break; - case 26: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[2] |= tmp << 0 | tmp << 8; - break; - case 27: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16; - break; - case 28: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - break; - case 29: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[3] |= tmp << 0; - break; - case 30: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[3] |= tmp << 0 | tmp << 8; - break; - case 31: buf0[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf0[3] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[0] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[1] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[2] |= tmp << 0 | tmp << 8 | tmp << 16 | tmp << 24; - buf1[3] |= tmp << 0 | tmp << 8 | tmp << 16; - break; - } - #endif - - out_len += p0; - - return out_len; -} - -inline u32 rule_op_mangle_dupechar_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if ( in_len == 0) return (in_len); - if ((in_len + p0) >= 32) return (in_len); - - const u32 in_len1 = in_len - 1; - - const u32 sh = (in_len1 & 3) * 8; - - u32 tmp = 0; - - switch (in_len1 / 4) - { - case 0: tmp = (buf0[0] >> sh) & 0xff; break; - case 1: tmp = (buf0[1] >> sh) & 0xff; break; - case 2: tmp = (buf0[2] >> sh) & 0xff; break; - case 3: tmp = (buf0[3] >> sh) & 0xff; break; - case 4: tmp = (buf1[0] >> sh) & 0xff; break; - case 5: tmp = (buf1[1] >> sh) & 0xff; break; - case 6: tmp = (buf1[2] >> sh) & 0xff; break; - case 7: tmp = (buf1[3] >> sh) & 0xff; break; - } - - u32 out_len = in_len; - - for (u32 i = 0; i < p0; i++) - { - append_block1 (out_len, buf0, buf1, tmp); - - out_len++; - } - - return out_len; -} - -inline u32 rule_op_mangle_dupechar_all (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if ( in_len == 0) return (in_len); - if ((in_len + in_len) >= 32) return (in_len); - - u32 out_len = in_len; - - u32 tib40[4]; - u32 tib41[4]; - - #ifdef IS_NV - tib40[0] = __byte_perm_S (buf0[0], 0, 0x1100); - tib40[1] = __byte_perm_S (buf0[0], 0, 0x3322); - tib40[2] = __byte_perm_S (buf0[1], 0, 0x1100); - tib40[3] = __byte_perm_S (buf0[1], 0, 0x3322); - tib41[0] = __byte_perm_S (buf0[2], 0, 0x1100); - tib41[1] = __byte_perm_S (buf0[2], 0, 0x3322); - tib41[2] = __byte_perm_S (buf0[3], 0, 0x1100); - tib41[3] = __byte_perm_S (buf0[3], 0, 0x3322); - - buf0[0] = tib40[0]; - buf0[1] = tib40[1]; - buf0[2] = tib40[2]; - buf0[3] = tib40[3]; - buf1[0] = tib41[0]; - buf1[1] = tib41[1]; - buf1[2] = tib41[2]; - buf1[3] = tib41[3]; - #endif - - #if defined IS_AMD || defined IS_GENERIC - tib40[0] = ((buf0[0] & 0x000000FF) << 0) | ((buf0[0] & 0x0000FF00) << 8); - tib40[1] = ((buf0[0] & 0x00FF0000) >> 16) | ((buf0[0] & 0xFF000000) >> 8); - tib40[2] = ((buf0[1] & 0x000000FF) << 0) | ((buf0[1] & 0x0000FF00) << 8); - tib40[3] = ((buf0[1] & 0x00FF0000) >> 16) | ((buf0[1] & 0xFF000000) >> 8); - tib41[0] = ((buf0[2] & 0x000000FF) << 0) | ((buf0[2] & 0x0000FF00) << 8); - tib41[1] = ((buf0[2] & 0x00FF0000) >> 16) | ((buf0[2] & 0xFF000000) >> 8); - tib41[2] = ((buf0[3] & 0x000000FF) << 0) | ((buf0[3] & 0x0000FF00) << 8); - tib41[3] = ((buf0[3] & 0x00FF0000) >> 16) | ((buf0[3] & 0xFF000000) >> 8); - - buf0[0] = tib40[0] | (tib40[0] << 8); - buf0[1] = tib40[1] | (tib40[1] << 8); - buf0[2] = tib40[2] | (tib40[2] << 8); - buf0[3] = tib40[3] | (tib40[3] << 8); - buf1[0] = tib41[0] | (tib41[0] << 8); - buf1[1] = tib41[1] | (tib41[1] << 8); - buf1[2] = tib41[2] | (tib41[2] << 8); - buf1[3] = tib41[3] | (tib41[3] << 8); - #endif - - out_len = out_len + out_len; - - return out_len; -} - -inline u32 rule_op_mangle_switch_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (in_len < 2) return (in_len); - - #ifdef IS_NV - buf0[0] = __byte_perm_S (buf0[0], 0, 0x3201); - #endif - - #if defined IS_AMD || defined IS_GENERIC - buf0[0] = (buf0[0] & 0xFFFF0000) | ((buf0[0] << 8) & 0x0000FF00) | ((buf0[0] >> 8) & 0x000000FF); - #endif - - return in_len; -} - -inline u32 rule_op_mangle_switch_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (in_len < 2) return (in_len); - - #ifdef IS_NV - switch (in_len) - { - case 2: buf0[0] = __byte_perm_S (buf0[0], 0, 0x5401); - break; - case 3: buf0[0] = __byte_perm_S (buf0[0], 0, 0x4120); - break; - case 4: buf0[0] = __byte_perm_S (buf0[0], 0, 0x2310); - break; - case 5: buf0[1] = __byte_perm_S (buf0[1], buf0[0], 0x7210); - buf0[0] = __byte_perm_S (buf0[0], buf0[1], 0x4210); - buf0[1] = __byte_perm_S (buf0[1], 0, 0x6543); - break; - case 6: buf0[1] = __byte_perm_S (buf0[1], 0, 0x5401); - break; - case 7: buf0[1] = __byte_perm_S (buf0[1], 0, 0x4120); - break; - case 8: buf0[1] = __byte_perm_S (buf0[1], 0, 0x2310); - break; - case 9: buf0[2] = __byte_perm_S (buf0[2], buf0[1], 0x7210); - buf0[1] = __byte_perm_S (buf0[1], buf0[2], 0x4210); - buf0[2] = __byte_perm_S (buf0[2], 0, 0x6543); - break; - case 10: buf0[2] = __byte_perm_S (buf0[2], 0, 0x5401); - break; - case 11: buf0[2] = __byte_perm_S (buf0[2], 0, 0x4120); - break; - case 12: buf0[2] = __byte_perm_S (buf0[2], 0, 0x2310); - break; - case 13: buf0[3] = __byte_perm_S (buf0[3], buf0[2], 0x7210); - buf0[2] = __byte_perm_S (buf0[2], buf0[3], 0x4210); - buf0[3] = __byte_perm_S (buf0[3], 0, 0x6543); - break; - case 14: buf0[3] = __byte_perm_S (buf0[3], 0, 0x5401); - break; - case 15: buf0[3] = __byte_perm_S (buf0[3], 0, 0x4120); - break; - case 16: buf0[3] = __byte_perm_S (buf0[3], 0, 0x2310); - break; - case 17: buf1[0] = __byte_perm_S (buf1[0], buf0[3], 0x7210); - buf0[3] = __byte_perm_S (buf0[3], buf1[0], 0x4210); - buf1[0] = __byte_perm_S (buf1[0], 0, 0x6543); - break; - case 18: buf1[0] = __byte_perm_S (buf1[0], 0, 0x5401); - break; - case 19: buf1[0] = __byte_perm_S (buf1[0], 0, 0x4120); - break; - case 20: buf1[0] = __byte_perm_S (buf1[0], 0, 0x2310); - break; - case 21: buf1[1] = __byte_perm_S (buf1[1], buf1[0], 0x7210); - buf1[0] = __byte_perm_S (buf1[0], buf1[1], 0x4210); - buf1[1] = __byte_perm_S (buf1[1], 0, 0x6543); - break; - case 22: buf1[1] = __byte_perm_S (buf1[1], 0, 0x5401); - break; - case 23: buf1[1] = __byte_perm_S (buf1[1], 0, 0x4120); - break; - case 24: buf1[1] = __byte_perm_S (buf1[1], 0, 0x2310); - break; - case 25: buf1[2] = __byte_perm_S (buf1[2], buf1[1], 0x7210); - buf1[1] = __byte_perm_S (buf1[1], buf1[2], 0x4210); - buf1[2] = __byte_perm_S (buf1[2], 0, 0x6543); - break; - case 26: buf1[2] = __byte_perm_S (buf1[2], 0, 0x5401); - break; - case 27: buf1[2] = __byte_perm_S (buf1[2], 0, 0x4120); - break; - case 28: buf1[2] = __byte_perm_S (buf1[2], 0, 0x2310); - break; - case 29: buf1[3] = __byte_perm_S (buf1[3], buf1[2], 0x7210); - buf1[2] = __byte_perm_S (buf1[2], buf1[3], 0x4210); - buf1[3] = __byte_perm_S (buf1[3], 0, 0x6543); - break; - case 30: buf1[3] = __byte_perm_S (buf1[3], 0, 0x5401); - break; - case 31: buf1[3] = __byte_perm_S (buf1[3], 0, 0x4120); - break; - } - #endif - - #if defined IS_AMD || defined IS_GENERIC - switch (in_len) - { - case 2: buf0[0] = ((buf0[0] << 8) & 0x0000FF00) | ((buf0[0] >> 8) & 0x000000FF); - break; - case 3: buf0[0] = (buf0[0] & 0x000000FF) | ((buf0[0] << 8) & 0x00FF0000) | ((buf0[0] >> 8) & 0x0000FF00); - break; - case 4: buf0[0] = (buf0[0] & 0x0000FFFF) | ((buf0[0] << 8) & 0xFF000000) | ((buf0[0] >> 8) & 0x00FF0000); - break; - case 5: buf0[1] = (buf0[0] & 0xFF000000) | buf0[1]; - buf0[0] = (buf0[0] & 0x00FFFFFF) | (buf0[1] << 24); - buf0[1] = (buf0[1] >> 24); - break; - case 6: buf0[1] = ((buf0[1] << 8) & 0x0000FF00) | ((buf0[1] >> 8) & 0x000000FF); - break; - case 7: buf0[1] = (buf0[1] & 0x000000FF) | ((buf0[1] << 8) & 0x00FF0000) | ((buf0[1] >> 8) & 0x0000FF00); - break; - case 8: buf0[1] = (buf0[1] & 0x0000FFFF) | ((buf0[1] << 8) & 0xFF000000) | ((buf0[1] >> 8) & 0x00FF0000); - break; - case 9: buf0[2] = (buf0[1] & 0xFF000000) | buf0[2]; - buf0[1] = (buf0[1] & 0x00FFFFFF) | (buf0[2] << 24); - buf0[2] = (buf0[2] >> 24); - break; - case 10: buf0[2] = ((buf0[2] << 8) & 0x0000FF00) | ((buf0[2] >> 8) & 0x000000FF); - break; - case 11: buf0[2] = (buf0[2] & 0x000000FF) | ((buf0[2] << 8) & 0x00FF0000) | ((buf0[2] >> 8) & 0x0000FF00); - break; - case 12: buf0[2] = (buf0[2] & 0x0000FFFF) | ((buf0[2] << 8) & 0xFF000000) | ((buf0[2] >> 8) & 0x00FF0000); - break; - case 13: buf0[3] = (buf0[2] & 0xFF000000) | buf0[3]; - buf0[2] = (buf0[2] & 0x00FFFFFF) | (buf0[3] << 24); - buf0[3] = (buf0[3] >> 24); - break; - case 14: buf0[3] = ((buf0[3] << 8) & 0x0000FF00) | ((buf0[3] >> 8) & 0x000000FF); - break; - case 15: buf0[3] = (buf0[3] & 0x000000FF) | ((buf0[3] << 8) & 0x00FF0000) | ((buf0[3] >> 8) & 0x0000FF00); - break; - case 16: buf0[3] = (buf0[3] & 0x0000FFFF) | ((buf0[3] << 8) & 0xFF000000) | ((buf0[3] >> 8) & 0x00FF0000); - break; - case 17: buf1[0] = (buf0[3] & 0xFF000000) | buf1[0]; - buf0[3] = (buf0[3] & 0x00FFFFFF) | (buf1[0] << 24); - buf1[0] = (buf1[0] >> 24); - break; - case 18: buf1[0] = ((buf1[0] << 8) & 0x0000FF00) | ((buf1[0] >> 8) & 0x000000FF); - break; - case 19: buf1[0] = (buf1[0] & 0x000000FF) | ((buf1[0] << 8) & 0x00FF0000) | ((buf1[0] >> 8) & 0x0000FF00); - break; - case 20: buf1[0] = (buf1[0] & 0x0000FFFF) | ((buf1[0] << 8) & 0xFF000000) | ((buf1[0] >> 8) & 0x00FF0000); - break; - case 21: buf1[1] = (buf1[0] & 0xFF000000) | buf1[1]; - buf1[0] = (buf1[0] & 0x00FFFFFF) | (buf1[1] << 24); - buf1[1] = (buf1[1] >> 24); - break; - case 22: buf1[1] = ((buf1[1] << 8) & 0x0000FF00) | ((buf1[1] >> 8) & 0x000000FF); - break; - case 23: buf1[1] = (buf1[1] & 0x000000FF) | ((buf1[1] << 8) & 0x00FF0000) | ((buf1[1] >> 8) & 0x0000FF00); - break; - case 24: buf1[1] = (buf1[1] & 0x0000FFFF) | ((buf1[1] << 8) & 0xFF000000) | ((buf1[1] >> 8) & 0x00FF0000); - break; - case 25: buf1[2] = (buf1[1] & 0xFF000000) | buf1[2]; - buf1[1] = (buf1[1] & 0x00FFFFFF) | (buf1[2] << 24); - buf1[2] = (buf1[2] >> 24); - break; - case 26: buf1[2] = ((buf1[2] << 8) & 0x0000FF00) | ((buf1[2] >> 8) & 0x000000FF); - break; - case 27: buf1[2] = (buf1[2] & 0x000000FF) | ((buf1[2] << 8) & 0x00FF0000) | ((buf1[2] >> 8) & 0x0000FF00); - break; - case 28: buf1[2] = (buf1[2] & 0x0000FFFF) | ((buf1[2] << 8) & 0xFF000000) | ((buf1[2] >> 8) & 0x00FF0000); - break; - case 29: buf1[3] = (buf1[2] & 0xFF000000) | buf1[3]; - buf1[2] = (buf1[2] & 0x00FFFFFF) | (buf1[3] << 24); - buf1[3] = (buf1[3] >> 24); - break; - case 30: buf1[3] = ((buf1[3] << 8) & 0x0000FF00) | ((buf1[3] >> 8) & 0x000000FF); - break; - case 31: buf1[3] = (buf1[3] & 0x000000FF) | ((buf1[3] << 8) & 0x00FF0000) | ((buf1[3] >> 8) & 0x0000FF00); - break; - } - #endif - - return in_len; -} - -inline u32 rule_op_mangle_switch_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - if (p1 >= in_len) return (in_len); - - u32 tmp0 = 0; - u32 tmp1 = 0; - - #ifdef IS_NV - switch (p0) - { - case 0: tmp0 = __byte_perm_S (buf0[0], 0, 0x6540); - break; - case 1: tmp0 = __byte_perm_S (buf0[0], 0, 0x6541); - break; - case 2: tmp0 = __byte_perm_S (buf0[0], 0, 0x6542); - break; - case 3: tmp0 = __byte_perm_S (buf0[0], 0, 0x6543); - break; - case 4: tmp0 = __byte_perm_S (buf0[1], 0, 0x6540); - break; - case 5: tmp0 = __byte_perm_S (buf0[1], 0, 0x6541); - break; - case 6: tmp0 = __byte_perm_S (buf0[1], 0, 0x6542); - break; - case 7: tmp0 = __byte_perm_S (buf0[1], 0, 0x6543); - break; - case 8: tmp0 = __byte_perm_S (buf0[2], 0, 0x6540); - break; - case 9: tmp0 = __byte_perm_S (buf0[2], 0, 0x6541); - break; - case 10: tmp0 = __byte_perm_S (buf0[2], 0, 0x6542); - break; - case 11: tmp0 = __byte_perm_S (buf0[2], 0, 0x6543); - break; - case 12: tmp0 = __byte_perm_S (buf0[3], 0, 0x6540); - break; - case 13: tmp0 = __byte_perm_S (buf0[3], 0, 0x6541); - break; - case 14: tmp0 = __byte_perm_S (buf0[3], 0, 0x6542); - break; - case 15: tmp0 = __byte_perm_S (buf0[3], 0, 0x6543); - break; - case 16: tmp0 = __byte_perm_S (buf1[0], 0, 0x6540); - break; - case 17: tmp0 = __byte_perm_S (buf1[0], 0, 0x6541); - break; - case 18: tmp0 = __byte_perm_S (buf1[0], 0, 0x6542); - break; - case 19: tmp0 = __byte_perm_S (buf1[0], 0, 0x6543); - break; - case 20: tmp0 = __byte_perm_S (buf1[1], 0, 0x6540); - break; - case 21: tmp0 = __byte_perm_S (buf1[1], 0, 0x6541); - break; - case 22: tmp0 = __byte_perm_S (buf1[1], 0, 0x6542); - break; - case 23: tmp0 = __byte_perm_S (buf1[1], 0, 0x6543); - break; - case 24: tmp0 = __byte_perm_S (buf1[2], 0, 0x6540); - break; - case 25: tmp0 = __byte_perm_S (buf1[2], 0, 0x6541); - break; - case 26: tmp0 = __byte_perm_S (buf1[2], 0, 0x6542); - break; - case 27: tmp0 = __byte_perm_S (buf1[2], 0, 0x6543); - break; - case 28: tmp0 = __byte_perm_S (buf1[3], 0, 0x6540); - break; - case 29: tmp0 = __byte_perm_S (buf1[3], 0, 0x6541); - break; - case 30: tmp0 = __byte_perm_S (buf1[3], 0, 0x6542); - break; - case 31: tmp0 = __byte_perm_S (buf1[3], 0, 0x6543); - break; - } - - switch (p1) - { - case 0: tmp1 = __byte_perm_S (buf0[0], 0, 0x6540); - buf0[0] = __byte_perm_S (tmp0, buf0[0], 0x7650); - break; - case 1: tmp1 = __byte_perm_S (buf0[0], 0, 0x6541); - buf0[0] = __byte_perm_S (tmp0, buf0[0], 0x7604); - break; - case 2: tmp1 = __byte_perm_S (buf0[0], 0, 0x6542); - buf0[0] = __byte_perm_S (tmp0, buf0[0], 0x7054); - break; - case 3: tmp1 = __byte_perm_S (buf0[0], 0, 0x6543); - buf0[0] = __byte_perm_S (tmp0, buf0[0], 0x0654); - break; - case 4: tmp1 = __byte_perm_S (buf0[1], 0, 0x6540); - buf0[1] = __byte_perm_S (tmp0, buf0[1], 0x7650); - break; - case 5: tmp1 = __byte_perm_S (buf0[1], 0, 0x6541); - buf0[1] = __byte_perm_S (tmp0, buf0[1], 0x7604); - break; - case 6: tmp1 = __byte_perm_S (buf0[1], 0, 0x6542); - buf0[1] = __byte_perm_S (tmp0, buf0[1], 0x7054); - break; - case 7: tmp1 = __byte_perm_S (buf0[1], 0, 0x6543); - buf0[1] = __byte_perm_S (tmp0, buf0[1], 0x0654); - break; - case 8: tmp1 = __byte_perm_S (buf0[2], 0, 0x6540); - buf0[2] = __byte_perm_S (tmp0, buf0[2], 0x7650); - break; - case 9: tmp1 = __byte_perm_S (buf0[2], 0, 0x6541); - buf0[2] = __byte_perm_S (tmp0, buf0[2], 0x7604); - break; - case 10: tmp1 = __byte_perm_S (buf0[2], 0, 0x6542); - buf0[2] = __byte_perm_S (tmp0, buf0[2], 0x7054); - break; - case 11: tmp1 = __byte_perm_S (buf0[2], 0, 0x6543); - buf0[2] = __byte_perm_S (tmp0, buf0[2], 0x0654); - break; - case 12: tmp1 = __byte_perm_S (buf0[3], 0, 0x6540); - buf0[3] = __byte_perm_S (tmp0, buf0[3], 0x7650); - break; - case 13: tmp1 = __byte_perm_S (buf0[3], 0, 0x6541); - buf0[3] = __byte_perm_S (tmp0, buf0[3], 0x7604); - break; - case 14: tmp1 = __byte_perm_S (buf0[3], 0, 0x6542); - buf0[3] = __byte_perm_S (tmp0, buf0[3], 0x7054); - break; - case 15: tmp1 = __byte_perm_S (buf0[3], 0, 0x6543); - buf0[3] = __byte_perm_S (tmp0, buf0[3], 0x0654); - break; - case 16: tmp1 = __byte_perm_S (buf1[0], 0, 0x6540); - buf1[0] = __byte_perm_S (tmp0, buf1[0], 0x7650); - break; - case 17: tmp1 = __byte_perm_S (buf1[0], 0, 0x6541); - buf1[0] = __byte_perm_S (tmp0, buf1[0], 0x7604); - break; - case 18: tmp1 = __byte_perm_S (buf1[0], 0, 0x6542); - buf1[0] = __byte_perm_S (tmp0, buf1[0], 0x7054); - break; - case 19: tmp1 = __byte_perm_S (buf1[0], 0, 0x6543); - buf1[0] = __byte_perm_S (tmp0, buf1[0], 0x0654); - break; - case 20: tmp1 = __byte_perm_S (buf1[1], 0, 0x6540); - buf1[1] = __byte_perm_S (tmp0, buf1[1], 0x7650); - break; - case 21: tmp1 = __byte_perm_S (buf1[1], 0, 0x6541); - buf1[1] = __byte_perm_S (tmp0, buf1[1], 0x7604); - break; - case 22: tmp1 = __byte_perm_S (buf1[1], 0, 0x6542); - buf1[1] = __byte_perm_S (tmp0, buf1[1], 0x7054); - break; - case 23: tmp1 = __byte_perm_S (buf1[1], 0, 0x6543); - buf1[1] = __byte_perm_S (tmp0, buf1[1], 0x0654); - break; - case 24: tmp1 = __byte_perm_S (buf1[2], 0, 0x6540); - buf1[2] = __byte_perm_S (tmp0, buf1[2], 0x7650); - break; - case 25: tmp1 = __byte_perm_S (buf1[2], 0, 0x6541); - buf1[2] = __byte_perm_S (tmp0, buf1[2], 0x7604); - break; - case 26: tmp1 = __byte_perm_S (buf1[2], 0, 0x6542); - buf1[2] = __byte_perm_S (tmp0, buf1[2], 0x7054); - break; - case 27: tmp1 = __byte_perm_S (buf1[2], 0, 0x6543); - buf1[2] = __byte_perm_S (tmp0, buf1[2], 0x0654); - break; - case 28: tmp1 = __byte_perm_S (buf1[3], 0, 0x6540); - buf1[3] = __byte_perm_S (tmp0, buf1[3], 0x7650); - break; - case 29: tmp1 = __byte_perm_S (buf1[3], 0, 0x6541); - buf1[3] = __byte_perm_S (tmp0, buf1[3], 0x7604); - break; - case 30: tmp1 = __byte_perm_S (buf1[3], 0, 0x6542); - buf1[3] = __byte_perm_S (tmp0, buf1[3], 0x7054); - break; - case 31: tmp1 = __byte_perm_S (buf1[3], 0, 0x6543); - buf1[3] = __byte_perm_S (tmp0, buf1[3], 0x0654); - break; - } - - switch (p0) - { - case 0: buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7650); - break; - case 1: buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7604); - break; - case 2: buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7054); - break; - case 3: buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x0654); - break; - case 4: buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7650); - break; - case 5: buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7604); - break; - case 6: buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7054); - break; - case 7: buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x0654); - break; - case 8: buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7650); - break; - case 9: buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7604); - break; - case 10: buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7054); - break; - case 11: buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x0654); - break; - case 12: buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7650); - break; - case 13: buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7604); - break; - case 14: buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7054); - break; - case 15: buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x0654); - break; - case 16: buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7650); - break; - case 17: buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7604); - break; - case 18: buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7054); - break; - case 19: buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x0654); - break; - case 20: buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7650); - break; - case 21: buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7604); - break; - case 22: buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7054); - break; - case 23: buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x0654); - break; - case 24: buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7650); - break; - case 25: buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7604); - break; - case 26: buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7054); - break; - case 27: buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x0654); - break; - case 28: buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7650); - break; - case 29: buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7604); - break; - case 30: buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7054); - break; - case 31: buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x0654); - break; - } - #endif - - #if defined IS_AMD || defined IS_GENERIC - switch (p0) - { - case 0: tmp0 = (buf0[0] >> 0) & 0xFF; - break; - case 1: tmp0 = (buf0[0] >> 8) & 0xFF; - break; - case 2: tmp0 = (buf0[0] >> 16) & 0xFF; - break; - case 3: tmp0 = (buf0[0] >> 24) & 0xFF; - break; - case 4: tmp0 = (buf0[1] >> 0) & 0xFF; - break; - case 5: tmp0 = (buf0[1] >> 8) & 0xFF; - break; - case 6: tmp0 = (buf0[1] >> 16) & 0xFF; - break; - case 7: tmp0 = (buf0[1] >> 24) & 0xFF; - break; - case 8: tmp0 = (buf0[2] >> 0) & 0xFF; - break; - case 9: tmp0 = (buf0[2] >> 8) & 0xFF; - break; - case 10: tmp0 = (buf0[2] >> 16) & 0xFF; - break; - case 11: tmp0 = (buf0[2] >> 24) & 0xFF; - break; - case 12: tmp0 = (buf0[3] >> 0) & 0xFF; - break; - case 13: tmp0 = (buf0[3] >> 8) & 0xFF; - break; - case 14: tmp0 = (buf0[3] >> 16) & 0xFF; - break; - case 15: tmp0 = (buf0[3] >> 24) & 0xFF; - break; - case 16: tmp0 = (buf1[0] >> 0) & 0xFF; - break; - case 17: tmp0 = (buf1[0] >> 8) & 0xFF; - break; - case 18: tmp0 = (buf1[0] >> 16) & 0xFF; - break; - case 19: tmp0 = (buf1[0] >> 24) & 0xFF; - break; - case 20: tmp0 = (buf1[1] >> 0) & 0xFF; - break; - case 21: tmp0 = (buf1[1] >> 8) & 0xFF; - break; - case 22: tmp0 = (buf1[1] >> 16) & 0xFF; - break; - case 23: tmp0 = (buf1[1] >> 24) & 0xFF; - break; - case 24: tmp0 = (buf1[2] >> 0) & 0xFF; - break; - case 25: tmp0 = (buf1[2] >> 8) & 0xFF; - break; - case 26: tmp0 = (buf1[2] >> 16) & 0xFF; - break; - case 27: tmp0 = (buf1[2] >> 24) & 0xFF; - break; - case 28: tmp0 = (buf1[3] >> 0) & 0xFF; - break; - case 29: tmp0 = (buf1[3] >> 8) & 0xFF; - break; - case 30: tmp0 = (buf1[3] >> 16) & 0xFF; - break; - case 31: tmp0 = (buf1[3] >> 24) & 0xFF; - break; - } - - switch (p1) - { - case 0: tmp1 = (buf0[0] >> 0) & 0xff; - buf0[0] = (buf0[0] & 0xffffff00) | tmp0 << 0; - break; - case 1: tmp1 = (buf0[0] >> 8) & 0xff; - buf0[0] = (buf0[0] & 0xffff00ff) | tmp0 << 8; - break; - case 2: tmp1 = (buf0[0] >> 16) & 0xff; - buf0[0] = (buf0[0] & 0xff00ffff) | tmp0 << 16; - break; - case 3: tmp1 = (buf0[0] >> 24) & 0xff; - buf0[0] = (buf0[0] & 0x00ffffff) | tmp0 << 24; - break; - case 4: tmp1 = (buf0[1] >> 0) & 0xff; - buf0[1] = (buf0[1] & 0xffffff00) | tmp0 << 0; - break; - case 5: tmp1 = (buf0[1] >> 8) & 0xff; - buf0[1] = (buf0[1] & 0xffff00ff) | tmp0 << 8; - break; - case 6: tmp1 = (buf0[1] >> 16) & 0xff; - buf0[1] = (buf0[1] & 0xff00ffff) | tmp0 << 16; - break; - case 7: tmp1 = (buf0[1] >> 24) & 0xff; - buf0[1] = (buf0[1] & 0x00ffffff) | tmp0 << 24; - break; - case 8: tmp1 = (buf0[2] >> 0) & 0xff; - buf0[2] = (buf0[2] & 0xffffff00) | tmp0 << 0; - break; - case 9: tmp1 = (buf0[2] >> 8) & 0xff; - buf0[2] = (buf0[2] & 0xffff00ff) | tmp0 << 8; - break; - case 10: tmp1 = (buf0[2] >> 16) & 0xff; - buf0[2] = (buf0[2] & 0xff00ffff) | tmp0 << 16; - break; - case 11: tmp1 = (buf0[2] >> 24) & 0xff; - buf0[2] = (buf0[2] & 0x00ffffff) | tmp0 << 24; - break; - case 12: tmp1 = (buf0[3] >> 0) & 0xff; - buf0[3] = (buf0[3] & 0xffffff00) | tmp0 << 0; - break; - case 13: tmp1 = (buf0[3] >> 8) & 0xff; - buf0[3] = (buf0[3] & 0xffff00ff) | tmp0 << 8; - break; - case 14: tmp1 = (buf0[3] >> 16) & 0xff; - buf0[3] = (buf0[3] & 0xff00ffff) | tmp0 << 16; - break; - case 15: tmp1 = (buf0[3] >> 24) & 0xff; - buf0[3] = (buf0[3] & 0x00ffffff) | tmp0 << 24; - break; - case 16: tmp1 = (buf1[0] >> 0) & 0xff; - buf1[0] = (buf1[0] & 0xffffff00) | tmp0 << 0; - break; - case 17: tmp1 = (buf1[0] >> 8) & 0xff; - buf1[0] = (buf1[0] & 0xffff00ff) | tmp0 << 8; - break; - case 18: tmp1 = (buf1[0] >> 16) & 0xff; - buf1[0] = (buf1[0] & 0xff00ffff) | tmp0 << 16; - break; - case 19: tmp1 = (buf1[0] >> 24) & 0xff; - buf1[0] = (buf1[0] & 0x00ffffff) | tmp0 << 24; - break; - case 20: tmp1 = (buf1[1] >> 0) & 0xff; - buf1[1] = (buf1[1] & 0xffffff00) | tmp0 << 0; - break; - case 21: tmp1 = (buf1[1] >> 8) & 0xff; - buf1[1] = (buf1[1] & 0xffff00ff) | tmp0 << 8; - break; - case 22: tmp1 = (buf1[1] >> 16) & 0xff; - buf1[1] = (buf1[1] & 0xff00ffff) | tmp0 << 16; - break; - case 23: tmp1 = (buf1[1] >> 24) & 0xff; - buf1[1] = (buf1[1] & 0x00ffffff) | tmp0 << 24; - break; - case 24: tmp1 = (buf1[2] >> 0) & 0xff; - buf1[2] = (buf1[2] & 0xffffff00) | tmp0 << 0; - break; - case 25: tmp1 = (buf1[2] >> 8) & 0xff; - buf1[2] = (buf1[2] & 0xffff00ff) | tmp0 << 8; - break; - case 26: tmp1 = (buf1[2] >> 16) & 0xff; - buf1[2] = (buf1[2] & 0xff00ffff) | tmp0 << 16; - break; - case 27: tmp1 = (buf1[2] >> 24) & 0xff; - buf1[2] = (buf1[2] & 0x00ffffff) | tmp0 << 24; - break; - case 28: tmp1 = (buf1[3] >> 0) & 0xff; - buf1[3] = (buf1[3] & 0xffffff00) | tmp0 << 0; - break; - case 29: tmp1 = (buf1[3] >> 8) & 0xff; - buf1[3] = (buf1[3] & 0xffff00ff) | tmp0 << 8; - break; - case 30: tmp1 = (buf1[3] >> 16) & 0xff; - buf1[3] = (buf1[3] & 0xff00ffff) | tmp0 << 16; - break; - case 31: tmp1 = (buf1[3] >> 24) & 0xff; - buf1[3] = (buf1[3] & 0x00ffffff) | tmp0 << 24; - break; - } - - switch (p0) - { - case 0: buf0[0] = (buf0[0] & 0xffffff00) | tmp1 << 0; - break; - case 1: buf0[0] = (buf0[0] & 0xffff00ff) | tmp1 << 8; - break; - case 2: buf0[0] = (buf0[0] & 0xff00ffff) | tmp1 << 16; - break; - case 3: buf0[0] = (buf0[0] & 0x00ffffff) | tmp1 << 24; - break; - case 4: buf0[1] = (buf0[1] & 0xffffff00) | tmp1 << 0; - break; - case 5: buf0[1] = (buf0[1] & 0xffff00ff) | tmp1 << 8; - break; - case 6: buf0[1] = (buf0[1] & 0xff00ffff) | tmp1 << 16; - break; - case 7: buf0[1] = (buf0[1] & 0x00ffffff) | tmp1 << 24; - break; - case 8: buf0[2] = (buf0[2] & 0xffffff00) | tmp1 << 0; - break; - case 9: buf0[2] = (buf0[2] & 0xffff00ff) | tmp1 << 8; - break; - case 10: buf0[2] = (buf0[2] & 0xff00ffff) | tmp1 << 16; - break; - case 11: buf0[2] = (buf0[2] & 0x00ffffff) | tmp1 << 24; - break; - case 12: buf0[3] = (buf0[3] & 0xffffff00) | tmp1 << 0; - break; - case 13: buf0[3] = (buf0[3] & 0xffff00ff) | tmp1 << 8; - break; - case 14: buf0[3] = (buf0[3] & 0xff00ffff) | tmp1 << 16; - break; - case 15: buf0[3] = (buf0[3] & 0x00ffffff) | tmp1 << 24; - break; - case 16: buf1[0] = (buf1[0] & 0xffffff00) | tmp1 << 0; - break; - case 17: buf1[0] = (buf1[0] & 0xffff00ff) | tmp1 << 8; - break; - case 18: buf1[0] = (buf1[0] & 0xff00ffff) | tmp1 << 16; - break; - case 19: buf1[0] = (buf1[0] & 0x00ffffff) | tmp1 << 24; - break; - case 20: buf1[1] = (buf1[1] & 0xffffff00) | tmp1 << 0; - break; - case 21: buf1[1] = (buf1[1] & 0xffff00ff) | tmp1 << 8; - break; - case 22: buf1[1] = (buf1[1] & 0xff00ffff) | tmp1 << 16; - break; - case 23: buf1[1] = (buf1[1] & 0x00ffffff) | tmp1 << 24; - break; - case 24: buf1[2] = (buf1[2] & 0xffffff00) | tmp1 << 0; - break; - case 25: buf1[2] = (buf1[2] & 0xffff00ff) | tmp1 << 8; - break; - case 26: buf1[2] = (buf1[2] & 0xff00ffff) | tmp1 << 16; - break; - case 27: buf1[2] = (buf1[2] & 0x00ffffff) | tmp1 << 24; - break; - case 28: buf1[3] = (buf1[3] & 0xffffff00) | tmp1 << 0; - break; - case 29: buf1[3] = (buf1[3] & 0xffff00ff) | tmp1 << 8; - break; - case 30: buf1[3] = (buf1[3] & 0xff00ffff) | tmp1 << 16; - break; - case 31: buf1[3] = (buf1[3] & 0x00ffffff) | tmp1 << 24; - break; - } - #endif - - return in_len; -} - -inline u32 rule_op_mangle_chr_shiftl (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - const u32 mr = 0xffu << ((p0 & 3) * 8); - const u32 ml = ~mr; - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) << 1) & mr); break; - case 1: buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) << 1) & mr); break; - case 2: buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) << 1) & mr); break; - case 3: buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) << 1) & mr); break; - case 4: buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) << 1) & mr); break; - case 5: buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) << 1) & mr); break; - case 6: buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) << 1) & mr); break; - case 7: buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) << 1) & mr); break; - } - - return in_len; -} - -inline u32 rule_op_mangle_chr_shiftr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - const u32 mr = 0xffu << ((p0 & 3) * 8); - const u32 ml = ~mr; - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) >> 1) & mr); break; - case 1: buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) >> 1) & mr); break; - case 2: buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) >> 1) & mr); break; - case 3: buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) >> 1) & mr); break; - case 4: buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) >> 1) & mr); break; - case 5: buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) >> 1) & mr); break; - case 6: buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) >> 1) & mr); break; - case 7: buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) >> 1) & mr); break; - } - - return in_len; -} - -inline u32 rule_op_mangle_chr_incr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - const u32 mr = 0xffu << ((p0 & 3) * 8); - const u32 ml = ~mr; - - const u32 n = 0x01010101 & mr; - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) + n) & mr); break; - case 1: buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) + n) & mr); break; - case 2: buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) + n) & mr); break; - case 3: buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) + n) & mr); break; - case 4: buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) + n) & mr); break; - case 5: buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) + n) & mr); break; - case 6: buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) + n) & mr); break; - case 7: buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) + n) & mr); break; - } - - return in_len; -} - -inline u32 rule_op_mangle_chr_decr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 >= in_len) return (in_len); - - const u32 mr = 0xffu << ((p0 & 3) * 8); - const u32 ml = ~mr; - - const u32 n = 0x01010101 & mr; - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) - n) & mr); break; - case 1: buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) - n) & mr); break; - case 2: buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) - n) & mr); break; - case 3: buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) - n) & mr); break; - case 4: buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) - n) & mr); break; - case 5: buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) - n) & mr); break; - case 6: buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) - n) & mr); break; - case 7: buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) - n) & mr); break; - } - - return in_len; -} - -inline u32 rule_op_mangle_replace_np1 (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if ((p0 + 1) >= in_len) return (in_len); - - u32 tib40[4]; - u32 tib41[4]; - - lshift_block (buf0, buf1, tib40, tib41); - - const u32 mr = 0xffu << ((p0 & 3) * 8); - const u32 ml = ~mr; - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & ml) | (tib40[0] & mr); break; - case 1: buf0[1] = (buf0[1] & ml) | (tib40[1] & mr); break; - case 2: buf0[2] = (buf0[2] & ml) | (tib40[2] & mr); break; - case 3: buf0[3] = (buf0[3] & ml) | (tib40[3] & mr); break; - case 4: buf1[0] = (buf1[0] & ml) | (tib41[0] & mr); break; - case 5: buf1[1] = (buf1[1] & ml) | (tib41[1] & mr); break; - case 6: buf1[2] = (buf1[2] & ml) | (tib41[2] & mr); break; - case 7: buf1[3] = (buf1[3] & ml) | (tib41[3] & mr); break; - } - - return in_len; -} - -inline u32 rule_op_mangle_replace_nm1 (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 == 0) return (in_len); - - if (p0 >= in_len) return (in_len); - - u32 tib40[4]; - u32 tib41[4]; - - rshift_block (buf0, buf1, tib40, tib41); - - const u32 mr = 0xffu << ((p0 & 3) * 8); - const u32 ml = ~mr; - - switch (p0 / 4) - { - case 0: buf0[0] = (buf0[0] & ml) | (tib40[0] & mr); break; - case 1: buf0[1] = (buf0[1] & ml) | (tib40[1] & mr); break; - case 2: buf0[2] = (buf0[2] & ml) | (tib40[2] & mr); break; - case 3: buf0[3] = (buf0[3] & ml) | (tib40[3] & mr); break; - case 4: buf1[0] = (buf1[0] & ml) | (tib41[0] & mr); break; - case 5: buf1[1] = (buf1[1] & ml) | (tib41[1] & mr); break; - case 6: buf1[2] = (buf1[2] & ml) | (tib41[2] & mr); break; - case 7: buf1[3] = (buf1[3] & ml) | (tib41[3] & mr); break; - } - - return in_len; -} - -inline u32 rule_op_mangle_dupeblock_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 > in_len) return (in_len); - - if ((in_len + p0) >= 32) return (in_len); - - u32 out_len = in_len; - - u32 tib40[4]; - u32 tib41[4]; - - tib40[0] = buf0[0]; - tib40[1] = buf0[1]; - tib40[2] = buf0[2]; - tib40[3] = buf0[3]; - tib41[0] = buf1[0]; - tib41[1] = buf1[1]; - tib41[2] = buf1[2]; - tib41[3] = buf1[3]; - - truncate_right (tib40, tib41, p0); - - rshift_block_N (buf0, buf1, buf0, buf1, p0); - - buf0[0] |= tib40[0]; - buf0[1] |= tib40[1]; - buf0[2] |= tib40[2]; - buf0[3] |= tib40[3]; - buf1[0] |= tib41[0]; - buf1[1] |= tib41[1]; - buf1[2] |= tib41[2]; - buf1[3] |= tib41[3]; - - out_len += p0; - - return out_len; -} - -inline u32 rule_op_mangle_dupeblock_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - if (p0 > in_len) return (in_len); - - if ((in_len + p0) >= 32) return (in_len); - - u32 out_len = in_len; - - u32 tib40[4]; - u32 tib41[4]; - - rshift_block_N (buf0, buf1, tib40, tib41, p0); - - truncate_left (tib40, tib41, out_len); - - buf0[0] |= tib40[0]; - buf0[1] |= tib40[1]; - buf0[2] |= tib40[2]; - buf0[3] |= tib40[3]; - buf1[0] |= tib41[0]; - buf1[1] |= tib41[1]; - buf1[2] |= tib41[2]; - buf1[3] |= tib41[3]; - - out_len += p0; - - return out_len; -} - -inline u32 rule_op_mangle_title (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - buf0[0] |= (generate_cmask (buf0[0])); - buf0[1] |= (generate_cmask (buf0[1])); - buf0[2] |= (generate_cmask (buf0[2])); - buf0[3] |= (generate_cmask (buf0[3])); - buf1[0] |= (generate_cmask (buf1[0])); - buf1[1] |= (generate_cmask (buf1[1])); - buf1[2] |= (generate_cmask (buf1[2])); - buf1[3] |= (generate_cmask (buf1[3])); - - #ifdef IS_NV - buf0[0] &= ~(0x00000020 & generate_cmask (buf0[0])); - - for (u32 i = 0; i < in_len; i++) - { - u32 tmp0; - u32 tmp1; - - switch (i) - { - case 0: tmp0 = __byte_perm_S (buf0[0], 0, 0x6540); - tmp1 = ~(0x00002000 & generate_cmask (buf0[0])); break; - case 1: tmp0 = __byte_perm_S (buf0[0], 0, 0x6541); - tmp1 = ~(0x00200000 & generate_cmask (buf0[0])); break; - case 2: tmp0 = __byte_perm_S (buf0[0], 0, 0x6542); - tmp1 = ~(0x20000000 & generate_cmask (buf0[0])); break; - case 3: tmp0 = __byte_perm_S (buf0[0], 0, 0x6543); - tmp1 = ~(0x00000020 & generate_cmask (buf0[1])); break; - case 4: tmp0 = __byte_perm_S (buf0[1], 0, 0x6540); - tmp1 = ~(0x00002000 & generate_cmask (buf0[1])); break; - case 5: tmp0 = __byte_perm_S (buf0[1], 0, 0x6541); - tmp1 = ~(0x00200000 & generate_cmask (buf0[1])); break; - case 6: tmp0 = __byte_perm_S (buf0[1], 0, 0x6542); - tmp1 = ~(0x20000000 & generate_cmask (buf0[1])); break; - case 7: tmp0 = __byte_perm_S (buf0[1], 0, 0x6543); - tmp1 = ~(0x00000020 & generate_cmask (buf0[2])); break; - case 8: tmp0 = __byte_perm_S (buf0[2], 0, 0x6540); - tmp1 = ~(0x00002000 & generate_cmask (buf0[2])); break; - case 9: tmp0 = __byte_perm_S (buf0[2], 0, 0x6541); - tmp1 = ~(0x00200000 & generate_cmask (buf0[2])); break; - case 10: tmp0 = __byte_perm_S (buf0[2], 0, 0x6542); - tmp1 = ~(0x20000000 & generate_cmask (buf0[2])); break; - case 11: tmp0 = __byte_perm_S (buf0[2], 0, 0x6543); - tmp1 = ~(0x00000020 & generate_cmask (buf0[3])); break; - case 12: tmp0 = __byte_perm_S (buf0[3], 0, 0x6540); - tmp1 = ~(0x00002000 & generate_cmask (buf0[3])); break; - case 13: tmp0 = __byte_perm_S (buf0[3], 0, 0x6541); - tmp1 = ~(0x00200000 & generate_cmask (buf0[3])); break; - case 14: tmp0 = __byte_perm_S (buf0[3], 0, 0x6542); - tmp1 = ~(0x20000000 & generate_cmask (buf0[3])); break; - case 15: tmp0 = __byte_perm_S (buf0[3], 0, 0x6543); - tmp1 = ~(0x00000020 & generate_cmask (buf1[0])); break; - case 16: tmp0 = __byte_perm_S (buf1[0], 0, 0x6540); - tmp1 = ~(0x00002000 & generate_cmask (buf1[0])); break; - case 17: tmp0 = __byte_perm_S (buf1[0], 0, 0x6541); - tmp1 = ~(0x00200000 & generate_cmask (buf1[0])); break; - case 18: tmp0 = __byte_perm_S (buf1[0], 0, 0x6542); - tmp1 = ~(0x20000000 & generate_cmask (buf1[0])); break; - case 19: tmp0 = __byte_perm_S (buf1[0], 0, 0x6543); - tmp1 = ~(0x00000020 & generate_cmask (buf1[1])); break; - case 20: tmp0 = __byte_perm_S (buf1[1], 0, 0x6540); - tmp1 = ~(0x00002000 & generate_cmask (buf1[1])); break; - case 21: tmp0 = __byte_perm_S (buf1[1], 0, 0x6541); - tmp1 = ~(0x00200000 & generate_cmask (buf1[1])); break; - case 22: tmp0 = __byte_perm_S (buf1[1], 0, 0x6542); - tmp1 = ~(0x20000000 & generate_cmask (buf1[1])); break; - case 23: tmp0 = __byte_perm_S (buf1[1], 0, 0x6543); - tmp1 = ~(0x00000020 & generate_cmask (buf1[2])); break; - case 24: tmp0 = __byte_perm_S (buf1[2], 0, 0x6540); - tmp1 = ~(0x00002000 & generate_cmask (buf1[2])); break; - case 25: tmp0 = __byte_perm_S (buf1[2], 0, 0x6541); - tmp1 = ~(0x00200000 & generate_cmask (buf1[2])); break; - case 26: tmp0 = __byte_perm_S (buf1[2], 0, 0x6542); - tmp1 = ~(0x20000000 & generate_cmask (buf1[2])); break; - case 27: tmp0 = __byte_perm_S (buf1[2], 0, 0x6543); - tmp1 = ~(0x00000020 & generate_cmask (buf1[3])); break; - case 28: tmp0 = __byte_perm_S (buf1[3], 0, 0x6540); - tmp1 = ~(0x00002000 & generate_cmask (buf1[3])); break; - case 29: tmp0 = __byte_perm_S (buf1[3], 0, 0x6541); - tmp1 = ~(0x00200000 & generate_cmask (buf1[3])); break; - case 30: tmp0 = __byte_perm_S (buf1[3], 0, 0x6542); - tmp1 = ~(0x20000000 & generate_cmask (buf1[3])); break; - } - - if (i < 3) - { - if (tmp0 == ' ') buf0[0] &= tmp1 ; - } - else if (i < 7) - { - if (tmp0 == ' ') buf0[1] &= tmp1 ; - } - else if (i < 11) - { - if (tmp0 == ' ') buf0[2] &= tmp1 ; - } - else if (i < 15) - { - if (tmp0 == ' ') buf0[3] &= tmp1 ; - } - else if (i < 19) - { - if (tmp0 == ' ') buf1[0] &= tmp1 ; - } - else if (i < 23) - { - if (tmp0 == ' ') buf1[1] &= tmp1 ; - } - else if (i < 27) - { - if (tmp0 == ' ') buf1[2] &= tmp1 ; - } - else if (i < 31) - { - if (tmp0 == ' ') buf1[3] &= tmp1 ; - } - } - #endif - - #if defined IS_AMD || defined IS_GENERIC - u32 tib40[4]; - u32 tib41[4]; - - const uchar4 tmp0 = (uchar4) (' '); - const uchar4 tmp1 = (uchar4) (0x00); - const uchar4 tmp2 = (uchar4) (0xff); - - uchar4 tmp; - - tmp = as_uchar4 (buf0[0]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[0] = as_uint (tmp); - tmp = as_uchar4 (buf0[1]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[1] = as_uint (tmp); - tmp = as_uchar4 (buf0[2]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[2] = as_uint (tmp); - tmp = as_uchar4 (buf0[3]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[3] = as_uint (tmp); - tmp = as_uchar4 (buf1[0]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[0] = as_uint (tmp); - tmp = as_uchar4 (buf1[1]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[1] = as_uint (tmp); - tmp = as_uchar4 (buf1[2]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[2] = as_uint (tmp); - tmp = as_uchar4 (buf1[3]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[3] = as_uint (tmp); - - rshift_block (tib40, tib41, tib40, tib41); tib40[0] |= 0xff; - - buf0[0] &= ~(generate_cmask (buf0[0]) & tib40[0]); - buf0[1] &= ~(generate_cmask (buf0[1]) & tib40[1]); - buf0[2] &= ~(generate_cmask (buf0[2]) & tib40[2]); - buf0[3] &= ~(generate_cmask (buf0[3]) & tib40[3]); - buf1[0] &= ~(generate_cmask (buf1[0]) & tib41[0]); - buf1[1] &= ~(generate_cmask (buf1[1]) & tib41[1]); - buf1[2] &= ~(generate_cmask (buf1[2]) & tib41[2]); - buf1[3] &= ~(generate_cmask (buf1[3]) & tib41[3]); - #endif - - return in_len; -} - -inline u32 apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len) -{ - u32 out_len = in_len; - - switch (name) - { - case RULE_OP_MANGLE_LREST: out_len = rule_op_mangle_lrest (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_UREST: out_len = rule_op_mangle_urest (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_LREST_UFIRST: out_len = rule_op_mangle_lrest_ufirst (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_UREST_LFIRST: out_len = rule_op_mangle_urest_lfirst (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_TREST: out_len = rule_op_mangle_trest (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_TOGGLE_AT: out_len = rule_op_mangle_toggle_at (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_REVERSE: out_len = rule_op_mangle_reverse (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DUPEWORD: out_len = rule_op_mangle_dupeword (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DUPEWORD_TIMES: out_len = rule_op_mangle_dupeword_times (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_REFLECT: out_len = rule_op_mangle_reflect (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_APPEND: out_len = rule_op_mangle_append (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_PREPEND: out_len = rule_op_mangle_prepend (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_ROTATE_LEFT: out_len = rule_op_mangle_rotate_left (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_ROTATE_RIGHT: out_len = rule_op_mangle_rotate_right (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DELETE_FIRST: out_len = rule_op_mangle_delete_first (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DELETE_LAST: out_len = rule_op_mangle_delete_last (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DELETE_AT: out_len = rule_op_mangle_delete_at (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_EXTRACT: out_len = rule_op_mangle_extract (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_OMIT: out_len = rule_op_mangle_omit (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_INSERT: out_len = rule_op_mangle_insert (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_OVERSTRIKE: out_len = rule_op_mangle_overstrike (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_TRUNCATE_AT: out_len = rule_op_mangle_truncate_at (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_REPLACE: out_len = rule_op_mangle_replace (p0, p1, buf0, buf1, out_len); break; - //case RULE_OP_MANGLE_PURGECHAR: out_len = rule_op_mangle_purgechar (p0, p1, buf0, buf1, out_len); break; - //case RULE_OP_MANGLE_TOGGLECASE_REC: out_len = rule_op_mangle_togglecase_rec (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DUPECHAR_FIRST: out_len = rule_op_mangle_dupechar_first (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DUPECHAR_LAST: out_len = rule_op_mangle_dupechar_last (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DUPECHAR_ALL: out_len = rule_op_mangle_dupechar_all (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_SWITCH_FIRST: out_len = rule_op_mangle_switch_first (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_SWITCH_LAST: out_len = rule_op_mangle_switch_last (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_SWITCH_AT: out_len = rule_op_mangle_switch_at (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_CHR_SHIFTL: out_len = rule_op_mangle_chr_shiftl (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_CHR_SHIFTR: out_len = rule_op_mangle_chr_shiftr (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_CHR_INCR: out_len = rule_op_mangle_chr_incr (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_CHR_DECR: out_len = rule_op_mangle_chr_decr (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_REPLACE_NP1: out_len = rule_op_mangle_replace_np1 (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_REPLACE_NM1: out_len = rule_op_mangle_replace_nm1 (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DUPEBLOCK_FIRST: out_len = rule_op_mangle_dupeblock_first (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_DUPEBLOCK_LAST: out_len = rule_op_mangle_dupeblock_last (p0, p1, buf0, buf1, out_len); break; - case RULE_OP_MANGLE_TITLE: out_len = rule_op_mangle_title (p0, p1, buf0, buf1, out_len); break; - } - - return out_len; -} - -inline u32 apply_rules (const __global u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len) -{ - u32 out_len = len; - - for (u32 i = 0; cmds[i] != 0; i++) - { - const u32 cmd = cmds[i]; - - const u32 name = (cmd >> 0) & 0xff; - const u32 p0 = (cmd >> 8) & 0xff; - const u32 p1 = (cmd >> 16) & 0xff; - - out_len = apply_rule (name, p0, p1, buf0, buf1, out_len); - } - - return out_len; -} - -inline u32x apply_rules_vect (const u32 pw_buf0[4], const u32 pw_buf1[4], const u32 pw_len, const __global kernel_rule_t *rules_buf, const u32 il_pos, u32x w0[4], u32x w1[4]) -{ - #if VECT_SIZE == 1 - - w0[0] = pw_buf0[0]; - w0[1] = pw_buf0[1]; - w0[2] = pw_buf0[2]; - w0[3] = pw_buf0[3]; - w1[0] = pw_buf1[0]; - w1[1] = pw_buf1[1]; - w1[2] = pw_buf1[2]; - w1[3] = pw_buf1[3]; - - return apply_rules (rules_buf[il_pos].cmds, w0, w1, pw_len); - - #else - - u32x out_len = 0; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < VECT_SIZE; i++) - { - u32 tmp0[4]; - u32 tmp1[4]; - - tmp0[0] = pw_buf0[0]; - tmp0[1] = pw_buf0[1]; - tmp0[2] = pw_buf0[2]; - tmp0[3] = pw_buf0[3]; - tmp1[0] = pw_buf1[0]; - tmp1[1] = pw_buf1[1]; - tmp1[2] = pw_buf1[2]; - tmp1[3] = pw_buf1[3]; - - const u32 tmp_len = apply_rules (rules_buf[il_pos + i].cmds, tmp0, tmp1, pw_len); - - switch (i) - { - #if VECT_SIZE >= 2 - case 0: - w0[0].s0 = tmp0[0]; - w0[1].s0 = tmp0[1]; - w0[2].s0 = tmp0[2]; - w0[3].s0 = tmp0[3]; - w1[0].s0 = tmp1[0]; - w1[1].s0 = tmp1[1]; - w1[2].s0 = tmp1[2]; - w1[3].s0 = tmp1[3]; - out_len.s0 = tmp_len; - break; - - case 1: - w0[0].s1 = tmp0[0]; - w0[1].s1 = tmp0[1]; - w0[2].s1 = tmp0[2]; - w0[3].s1 = tmp0[3]; - w1[0].s1 = tmp1[0]; - w1[1].s1 = tmp1[1]; - w1[2].s1 = tmp1[2]; - w1[3].s1 = tmp1[3]; - out_len.s1 = tmp_len; - break; - #endif - - #if VECT_SIZE >= 4 - case 2: - w0[0].s2 = tmp0[0]; - w0[1].s2 = tmp0[1]; - w0[2].s2 = tmp0[2]; - w0[3].s2 = tmp0[3]; - w1[0].s2 = tmp1[0]; - w1[1].s2 = tmp1[1]; - w1[2].s2 = tmp1[2]; - w1[3].s2 = tmp1[3]; - out_len.s2 = tmp_len; - break; - - case 3: - w0[0].s3 = tmp0[0]; - w0[1].s3 = tmp0[1]; - w0[2].s3 = tmp0[2]; - w0[3].s3 = tmp0[3]; - w1[0].s3 = tmp1[0]; - w1[1].s3 = tmp1[1]; - w1[2].s3 = tmp1[2]; - w1[3].s3 = tmp1[3]; - out_len.s3 = tmp_len; - break; - #endif - - #if VECT_SIZE >= 8 - case 4: - w0[0].s4 = tmp0[0]; - w0[1].s4 = tmp0[1]; - w0[2].s4 = tmp0[2]; - w0[3].s4 = tmp0[3]; - w1[0].s4 = tmp1[0]; - w1[1].s4 = tmp1[1]; - w1[2].s4 = tmp1[2]; - w1[3].s4 = tmp1[3]; - out_len.s4 = tmp_len; - break; - - case 5: - w0[0].s5 = tmp0[0]; - w0[1].s5 = tmp0[1]; - w0[2].s5 = tmp0[2]; - w0[3].s5 = tmp0[3]; - w1[0].s5 = tmp1[0]; - w1[1].s5 = tmp1[1]; - w1[2].s5 = tmp1[2]; - w1[3].s5 = tmp1[3]; - out_len.s5 = tmp_len; - break; - - case 6: - w0[0].s6 = tmp0[0]; - w0[1].s6 = tmp0[1]; - w0[2].s6 = tmp0[2]; - w0[3].s6 = tmp0[3]; - w1[0].s6 = tmp1[0]; - w1[1].s6 = tmp1[1]; - w1[2].s6 = tmp1[2]; - w1[3].s6 = tmp1[3]; - out_len.s6 = tmp_len; - break; - - case 7: - w0[0].s7 = tmp0[0]; - w0[1].s7 = tmp0[1]; - w0[2].s7 = tmp0[2]; - w0[3].s7 = tmp0[3]; - w1[0].s7 = tmp1[0]; - w1[1].s7 = tmp1[1]; - w1[2].s7 = tmp1[2]; - w1[3].s7 = tmp1[3]; - out_len.s7 = tmp_len; - break; - #endif - - #if VECT_SIZE >= 16 - case 8: - w0[0].s8 = tmp0[0]; - w0[1].s8 = tmp0[1]; - w0[2].s8 = tmp0[2]; - w0[3].s8 = tmp0[3]; - w1[0].s8 = tmp1[0]; - w1[1].s8 = tmp1[1]; - w1[2].s8 = tmp1[2]; - w1[3].s8 = tmp1[3]; - out_len.s8 = tmp_len; - break; - - case 9: - w0[0].s9 = tmp0[0]; - w0[1].s9 = tmp0[1]; - w0[2].s9 = tmp0[2]; - w0[3].s9 = tmp0[3]; - w1[0].s9 = tmp1[0]; - w1[1].s9 = tmp1[1]; - w1[2].s9 = tmp1[2]; - w1[3].s9 = tmp1[3]; - out_len.s9 = tmp_len; - break; - - case 10: - w0[0].sa = tmp0[0]; - w0[1].sa = tmp0[1]; - w0[2].sa = tmp0[2]; - w0[3].sa = tmp0[3]; - w1[0].sa = tmp1[0]; - w1[1].sa = tmp1[1]; - w1[2].sa = tmp1[2]; - w1[3].sa = tmp1[3]; - out_len.sa = tmp_len; - break; - - case 11: - w0[0].sb = tmp0[0]; - w0[1].sb = tmp0[1]; - w0[2].sb = tmp0[2]; - w0[3].sb = tmp0[3]; - w1[0].sb = tmp1[0]; - w1[1].sb = tmp1[1]; - w1[2].sb = tmp1[2]; - w1[3].sb = tmp1[3]; - out_len.sb = tmp_len; - break; - - case 12: - w0[0].sc = tmp0[0]; - w0[1].sc = tmp0[1]; - w0[2].sc = tmp0[2]; - w0[3].sc = tmp0[3]; - w1[0].sc = tmp1[0]; - w1[1].sc = tmp1[1]; - w1[2].sc = tmp1[2]; - w1[3].sc = tmp1[3]; - out_len.sc = tmp_len; - break; - - case 13: - w0[0].sd = tmp0[0]; - w0[1].sd = tmp0[1]; - w0[2].sd = tmp0[2]; - w0[3].sd = tmp0[3]; - w1[0].sd = tmp1[0]; - w1[1].sd = tmp1[1]; - w1[2].sd = tmp1[2]; - w1[3].sd = tmp1[3]; - out_len.sd = tmp_len; - break; - - case 14: - w0[0].se = tmp0[0]; - w0[1].se = tmp0[1]; - w0[2].se = tmp0[2]; - w0[3].se = tmp0[3]; - w1[0].se = tmp1[0]; - w1[1].se = tmp1[1]; - w1[2].se = tmp1[2]; - w1[3].se = tmp1[3]; - out_len.se = tmp_len; - break; - - case 15: - w0[0].sf = tmp0[0]; - w0[1].sf = tmp0[1]; - w0[2].sf = tmp0[2]; - w0[3].sf = tmp0[3]; - w1[0].sf = tmp1[0]; - w1[1].sf = tmp1[1]; - w1[2].sf = tmp1[2]; - w1[3].sf = tmp1[3]; - out_len.sf = tmp_len; - break; - #endif - } - } - - return out_len; - - #endif -} diff --git a/OpenCL/simd.c b/OpenCL/simd.c deleted file mode 100644 index 882e320..0000000 --- a/OpenCL/simd.c +++ /dev/null @@ -1,1164 +0,0 @@ -/** - * Authors.....: Jens Steube - * magnum - * - * License.....: MIT - */ - -// vliw1 - -#if VECT_SIZE == 1 - -#define MATCHES_ONE_VV(a,b) ((a) == (b)) -#define MATCHES_ONE_VS(a,b) ((a) == (b)) - -#define COMPARE_S_SIMD(h0,h1,h2,h3) \ -{ \ - if (((h0) == search[0]) && ((h1) == search[1]) && ((h2) == search[2]) && ((h3) == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos); \ - } \ - } \ -} - -#define COMPARE_M_SIMD(h0,h1,h2,h3) \ -{ \ - const u32 digest_tp0[4] = { h0, h1, h2, h3 }; \ - \ - if (check (digest_tp0, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (atomic_inc (&hashes_shown[final_hash_pos]) == 0) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos); \ - } \ - } \ - } \ -} - -#endif - -// vliw2 - -#define vector_accessible(p,c,e) (((p) + (e)) < (c)) - -#if VECT_SIZE == 2 - -#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1)) -#define MATCHES_ONE_VS(a,b) (((a).s0 == (b) ) || ((a).s1 == (b) )) - -#define COMPARE_S_SIMD(h0,h1,h2,h3) \ -{ \ - if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0); \ - } \ - } \ - \ - if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1); \ - } \ - } \ -} - -#define COMPARE_M_SIMD(h0,h1,h2,h3) \ -{ \ - const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 }; \ - const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 }; \ - \ - if (check (digest_tp0, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0); \ - } \ - } \ - } \ - \ - if (check (digest_tp1, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1); \ - } \ - } \ - } \ -} - -#endif - -// vliw4 - -#if VECT_SIZE == 4 - -#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3)) -#define MATCHES_ONE_VS(a,b) (((a).s0 == (b) ) || ((a).s1 == (b) ) || ((a).s2 == (b) ) || ((a).s3 == (b) )) - -#define COMPARE_S_SIMD(h0,h1,h2,h3) \ -{ \ - if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0); \ - } \ - } \ - \ - if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1); \ - } \ - } \ - \ - if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2); \ - } \ - } \ - \ - if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3); \ - } \ - } \ -} - -#define COMPARE_M_SIMD(h0,h1,h2,h3) \ -{ \ - const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 }; \ - const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 }; \ - const u32 digest_tp2[4] = { h0.s2, h1.s2, h2.s2, h3.s2 }; \ - const u32 digest_tp3[4] = { h0.s3, h1.s3, h2.s3, h3.s3 }; \ - \ - if (check (digest_tp0, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0); \ - } \ - } \ - } \ - \ - if (check (digest_tp1, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1); \ - } \ - } \ - } \ - \ - if (check (digest_tp2, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp2, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2); \ - } \ - } \ - } \ - \ - if (check (digest_tp3, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp3, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3); \ - } \ - } \ - } \ -} - -#endif - -// vliw8 - -#if VECT_SIZE == 8 - -#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3) || ((a).s4 == (b).s4) || ((a).s5 == (b).s5) || ((a).s6 == (b).s6) || ((a).s7 == (b).s7)) -#define MATCHES_ONE_VS(a,b) (((a).s0 == (b) ) || ((a).s1 == (b) ) || ((a).s2 == (b) ) || ((a).s3 == (b) ) || ((a).s4 == (b) ) || ((a).s5 == (b) ) || ((a).s6 == (b) ) || ((a).s7 == (b) )) - -#define COMPARE_S_SIMD(h0,h1,h2,h3) \ -{ \ - if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0); \ - } \ - } \ - \ - if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1); \ - } \ - } \ - \ - if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2); \ - } \ - } \ - \ - if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3); \ - } \ - } \ - if (((h0).s4 == search[0]) && ((h1).s4 == search[1]) && ((h2).s4 == search[2]) && ((h3).s4 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 4); \ - } \ - } \ - \ - if (((h0).s5 == search[0]) && ((h1).s5 == search[1]) && ((h2).s5 == search[2]) && ((h3).s5 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 5); \ - } \ - } \ - \ - if (((h0).s6 == search[0]) && ((h1).s6 == search[1]) && ((h2).s6 == search[2]) && ((h3).s6 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 6); \ - } \ - } \ - \ - if (((h0).s7 == search[0]) && ((h1).s7 == search[1]) && ((h2).s7 == search[2]) && ((h3).s7 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 7); \ - } \ - } \ -} - -#define COMPARE_M_SIMD(h0,h1,h2,h3) \ -{ \ - const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 }; \ - const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 }; \ - const u32 digest_tp2[4] = { h0.s2, h1.s2, h2.s2, h3.s2 }; \ - const u32 digest_tp3[4] = { h0.s3, h1.s3, h2.s3, h3.s3 }; \ - const u32 digest_tp4[4] = { h0.s4, h1.s4, h2.s4, h3.s4 }; \ - const u32 digest_tp5[4] = { h0.s5, h1.s5, h2.s5, h3.s5 }; \ - const u32 digest_tp6[4] = { h0.s6, h1.s6, h2.s6, h3.s6 }; \ - const u32 digest_tp7[4] = { h0.s7, h1.s7, h2.s7, h3.s7 }; \ - \ - if (check (digest_tp0, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0); \ - } \ - } \ - } \ - \ - if (check (digest_tp1, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1); \ - } \ - } \ - } \ - \ - if (check (digest_tp2, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp2, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2); \ - } \ - } \ - } \ - \ - if (check (digest_tp3, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp3, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3); \ - } \ - } \ - } \ - if (check (digest_tp4, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp4, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 4); \ - } \ - } \ - } \ - \ - if (check (digest_tp5, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp5, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 5); \ - } \ - } \ - } \ - \ - if (check (digest_tp6, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp6, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 6); \ - } \ - } \ - } \ - \ - if (check (digest_tp7, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp7, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 7); \ - } \ - } \ - } \ -} - -#endif - -// vliw16 - -#if VECT_SIZE == 16 - -#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3) || ((a).s4 == (b).s4) || ((a).s5 == (b).s5) || ((a).s6 == (b).s6) || ((a).s7 == (b).s7) || ((a).s8 == (b).s8) || ((a).s9 == (b).s9) || ((a).sa == (b).sa) || ((a).sb == (b).sb) || ((a).sc == (b).sc) || ((a).sd == (b).sd) || ((a).se == (b).se) || ((a).sf == (b).sf)) -#define MATCHES_ONE_VS(a,b) (((a).s0 == (b) ) || ((a).s1 == (b) ) || ((a).s2 == (b) ) || ((a).s3 == (b) ) || ((a).s4 == (b) ) || ((a).s5 == (b) ) || ((a).s6 == (b) ) || ((a).s7 == (b) ) || ((a).s8 == (b) ) || ((a).s9 == (b) ) || ((a).sa == (b) ) || ((a).sb == (b) ) || ((a).sc == (b) ) || ((a).sd == (b) ) || ((a).se == (b) ) || ((a).sf == (b) )) - -#define COMPARE_S_SIMD(h0,h1,h2,h3) \ -{ \ - if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0); \ - } \ - } \ - \ - if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1); \ - } \ - } \ - \ - if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2); \ - } \ - } \ - \ - if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3); \ - } \ - } \ - if (((h0).s4 == search[0]) && ((h1).s4 == search[1]) && ((h2).s4 == search[2]) && ((h3).s4 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 4); \ - } \ - } \ - \ - if (((h0).s5 == search[0]) && ((h1).s5 == search[1]) && ((h2).s5 == search[2]) && ((h3).s5 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 5); \ - } \ - } \ - \ - if (((h0).s6 == search[0]) && ((h1).s6 == search[1]) && ((h2).s6 == search[2]) && ((h3).s6 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 6); \ - } \ - } \ - \ - if (((h0).s7 == search[0]) && ((h1).s7 == search[1]) && ((h2).s7 == search[2]) && ((h3).s7 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 7); \ - } \ - } \ - \ - if (((h0).s8 == search[0]) && ((h1).s8 == search[1]) && ((h2).s8 == search[2]) && ((h3).s8 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 8) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 8); \ - } \ - } \ - \ - if (((h0).s9 == search[0]) && ((h1).s9 == search[1]) && ((h2).s9 == search[2]) && ((h3).s9 == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 9) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 9); \ - } \ - } \ - \ - if (((h0).sa == search[0]) && ((h1).sa == search[1]) && ((h2).sa == search[2]) && ((h3).sa == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 10) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 10); \ - } \ - } \ - \ - if (((h0).sb == search[0]) && ((h1).sb == search[1]) && ((h2).sb == search[2]) && ((h3).sb == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 11) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 11); \ - } \ - } \ - \ - if (((h0).sc == search[0]) && ((h1).sc == search[1]) && ((h2).sc == search[2]) && ((h3).sc == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 12) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 12); \ - } \ - } \ - \ - if (((h0).sd == search[0]) && ((h1).sd == search[1]) && ((h2).sd == search[2]) && ((h3).sd == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 13) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 13); \ - } \ - } \ - \ - if (((h0).se == search[0]) && ((h1).se == search[1]) && ((h2).se == search[2]) && ((h3).se == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 14) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 14); \ - } \ - } \ - \ - if (((h0).sf == search[0]) && ((h1).sf == search[1]) && ((h2).sf == search[2]) && ((h3).sf == search[3])) \ - { \ - const u32 final_hash_pos = digests_offset + 0; \ - \ - if (vector_accessible (il_pos, il_cnt, 15) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 15); \ - } \ - } \ -} - -#define COMPARE_M_SIMD(h0,h1,h2,h3) \ -{ \ - const u32 digest_tp00[4] = { h0.s0, h1.s0, h2.s0, h3.s0 }; \ - const u32 digest_tp01[4] = { h0.s1, h1.s1, h2.s1, h3.s1 }; \ - const u32 digest_tp02[4] = { h0.s2, h1.s2, h2.s2, h3.s2 }; \ - const u32 digest_tp03[4] = { h0.s3, h1.s3, h2.s3, h3.s3 }; \ - const u32 digest_tp04[4] = { h0.s4, h1.s4, h2.s4, h3.s4 }; \ - const u32 digest_tp05[4] = { h0.s5, h1.s5, h2.s5, h3.s5 }; \ - const u32 digest_tp06[4] = { h0.s6, h1.s6, h2.s6, h3.s6 }; \ - const u32 digest_tp07[4] = { h0.s7, h1.s7, h2.s7, h3.s7 }; \ - const u32 digest_tp08[4] = { h0.s8, h1.s8, h2.s8, h3.s8 }; \ - const u32 digest_tp09[4] = { h0.s9, h1.s9, h2.s9, h3.s9 }; \ - const u32 digest_tp10[4] = { h0.sa, h1.sa, h2.sa, h3.sa }; \ - const u32 digest_tp11[4] = { h0.sb, h1.sb, h2.sb, h3.sb }; \ - const u32 digest_tp12[4] = { h0.sc, h1.sc, h2.sc, h3.sc }; \ - const u32 digest_tp13[4] = { h0.sd, h1.sd, h2.sd, h3.sd }; \ - const u32 digest_tp14[4] = { h0.se, h1.se, h2.se, h3.se }; \ - const u32 digest_tp15[4] = { h0.sf, h1.sf, h2.sf, h3.sf }; \ - \ - if (check (digest_tp00, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp00, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0); \ - } \ - } \ - } \ - \ - if (check (digest_tp01, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp01, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1); \ - } \ - } \ - } \ - \ - if (check (digest_tp02, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp02, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2); \ - } \ - } \ - } \ - \ - if (check (digest_tp03, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp03, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3); \ - } \ - } \ - } \ - \ - if (check (digest_tp04, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp04, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 4); \ - } \ - } \ - } \ - \ - if (check (digest_tp05, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp05, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 5); \ - } \ - } \ - } \ - \ - if (check (digest_tp06, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp06, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 6); \ - } \ - } \ - } \ - \ - if (check (digest_tp07, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp07, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 7); \ - } \ - } \ - } \ - \ - if (check (digest_tp08, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp08, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 8) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 8); \ - } \ - } \ - } \ - \ - if (check (digest_tp09, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp09, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 9) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 9); \ - } \ - } \ - } \ - \ - if (check (digest_tp10, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp10, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 10) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 10); \ - } \ - } \ - } \ - \ - if (check (digest_tp11, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp11, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 11) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 11); \ - } \ - } \ - } \ - \ - if (check (digest_tp12, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp12, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 12) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 12); \ - } \ - } \ - } \ - \ - if (check (digest_tp13, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp13, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 13) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 13); \ - } \ - } \ - } \ - \ - if (check (digest_tp14, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp14, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 14) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 14); \ - } \ - } \ - } \ - \ - if (check (digest_tp15, \ - bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, \ - bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, \ - bitmap_mask, \ - bitmap_shift1, \ - bitmap_shift2)) \ - { \ - int digest_pos = find_hash (digest_tp15, digests_cnt, &digests_buf[digests_offset]); \ - \ - if (digest_pos != -1) \ - { \ - const u32 final_hash_pos = digests_offset + digest_pos; \ - \ - if (vector_accessible (il_pos, il_cnt, 15) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0)) \ - { \ - mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 15); \ - } \ - } \ - } \ -} - -#endif - -#define MATCHES_NONE_VV(a,b) !(MATCHES_ONE_VV ((a), (b))) -#define MATCHES_NONE_VS(a,b) !(MATCHES_ONE_VS ((a), (b))) - -// attack-mode 0 - -inline u32x ix_create_bft (__global bf_t *bfs_buf, const u32 il_pos) -{ - #if VECT_SIZE == 1 - const u32x ix = (u32x) (bfs_buf[il_pos + 0].i); - #elif VECT_SIZE == 2 - const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i); - #elif VECT_SIZE == 4 - const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i); - #elif VECT_SIZE == 8 - const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i, bfs_buf[il_pos + 4].i, bfs_buf[il_pos + 5].i, bfs_buf[il_pos + 6].i, bfs_buf[il_pos + 7].i); - #elif VECT_SIZE == 16 - const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i, bfs_buf[il_pos + 4].i, bfs_buf[il_pos + 5].i, bfs_buf[il_pos + 6].i, bfs_buf[il_pos + 7].i, bfs_buf[il_pos + 8].i, bfs_buf[il_pos + 9].i, bfs_buf[il_pos + 10].i, bfs_buf[il_pos + 11].i, bfs_buf[il_pos + 12].i, bfs_buf[il_pos + 13].i, bfs_buf[il_pos + 14].i, bfs_buf[il_pos + 15].i); - #endif - - return ix; -} - -// attack-mode 1 - -inline u32x pwlenx_create_combt (__global comb_t *combs_buf, const u32 il_pos) -{ - #if VECT_SIZE == 1 - const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len); - #elif VECT_SIZE == 2 - const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len); - #elif VECT_SIZE == 4 - const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len); - #elif VECT_SIZE == 8 - const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len, combs_buf[il_pos + 4].pw_len, combs_buf[il_pos + 5].pw_len, combs_buf[il_pos + 6].pw_len, combs_buf[il_pos + 7].pw_len); - #elif VECT_SIZE == 16 - const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len, combs_buf[il_pos + 4].pw_len, combs_buf[il_pos + 5].pw_len, combs_buf[il_pos + 6].pw_len, combs_buf[il_pos + 7].pw_len, combs_buf[il_pos + 8].pw_len, combs_buf[il_pos + 9].pw_len, combs_buf[il_pos + 10].pw_len, combs_buf[il_pos + 11].pw_len, combs_buf[il_pos + 12].pw_len, combs_buf[il_pos + 13].pw_len, combs_buf[il_pos + 14].pw_len, combs_buf[il_pos + 15].pw_len); - #endif - - return pw_lenx; -} - -inline u32x ix_create_combt (__global comb_t *combs_buf, const u32 il_pos, const int idx) -{ - #if VECT_SIZE == 1 - const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx]); - #elif VECT_SIZE == 2 - const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx]); - #elif VECT_SIZE == 4 - const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx]); - #elif VECT_SIZE == 8 - const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx], combs_buf[il_pos + 4].i[idx], combs_buf[il_pos + 5].i[idx], combs_buf[il_pos + 6].i[idx], combs_buf[il_pos + 7].i[idx]); - #elif VECT_SIZE == 16 - const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx], combs_buf[il_pos + 4].i[idx], combs_buf[il_pos + 5].i[idx], combs_buf[il_pos + 6].i[idx], combs_buf[il_pos + 7].i[idx], combs_buf[il_pos + 8].i[idx], combs_buf[il_pos + 9].i[idx], combs_buf[il_pos + 10].i[idx], combs_buf[il_pos + 11].i[idx], combs_buf[il_pos + 12].i[idx], combs_buf[il_pos + 13].i[idx], combs_buf[il_pos + 14].i[idx], combs_buf[il_pos + 15].i[idx]); - #endif - - return ix; -} - -#if VECT_SIZE == 1 -#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 1) + 0].var[(idx)]) -#elif VECT_SIZE == 2 -#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 2) + 0].var[(idx)], (arr)[((gid) * 2) + 1].var[(idx)]) -#elif VECT_SIZE == 4 -#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 4) + 0].var[(idx)], (arr)[((gid) * 4) + 1].var[(idx)], (arr)[((gid) * 4) + 2].var[(idx)], (arr)[((gid) * 4) + 3].var[(idx)]) -#elif VECT_SIZE == 8 -#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 8) + 0].var[(idx)], (arr)[((gid) * 8) + 1].var[(idx)], (arr)[((gid) * 8) + 2].var[(idx)], (arr)[((gid) * 8) + 3].var[(idx)], (arr)[((gid) * 8) + 4].var[(idx)], (arr)[((gid) * 8) + 5].var[(idx)], (arr)[((gid) * 8) + 6].var[(idx)], (arr)[((gid) * 8) + 7].var[(idx)]) -#elif VECT_SIZE == 16 -#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 16) + 0].var[(idx)], (arr)[((gid) * 16) + 1].var[(idx)], (arr)[((gid) * 16) + 2].var[(idx)], (arr)[((gid) * 16) + 3].var[(idx)], (arr)[((gid) * 16) + 4].var[(idx)], (arr)[((gid) * 16) + 5].var[(idx)], (arr)[((gid) * 16) + 6].var[(idx)], (arr)[((gid) * 16) + 7].var[(idx)], (arr)[((gid) * 16) + 8].var[(idx)], (arr)[((gid) * 16) + 9].var[(idx)], (arr)[((gid) * 16) + 10].var[(idx)], (arr)[((gid) * 16) + 11].var[(idx)], (arr)[((gid) * 16) + 12].var[(idx)], (arr)[((gid) * 16) + 13].var[(idx)], (arr)[((gid) * 16) + 14].var[(idx)], (arr)[((gid) * 16) + 15].var[(idx)]) -#endif - -#if VECT_SIZE == 1 -#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 1) + 0].var) -#elif VECT_SIZE == 2 -#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 2) + 0].var, (arr)[((gid) * 2) + 1].var) -#elif VECT_SIZE == 4 -#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 4) + 0].var, (arr)[((gid) * 4) + 1].var, (arr)[((gid) * 4) + 2].var, (arr)[((gid) * 4) + 3].var) -#elif VECT_SIZE == 8 -#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 8) + 0].var, (arr)[((gid) * 8) + 1].var, (arr)[((gid) * 8) + 2].var, (arr)[((gid) * 8) + 3].var, (arr)[((gid) * 8) + 4].var, (arr)[((gid) * 8) + 5].var, (arr)[((gid) * 8) + 6].var, (arr)[((gid) * 8) + 7].var) -#elif VECT_SIZE == 16 -#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 16) + 0].var, (arr)[((gid) * 16) + 1].var, (arr)[((gid) * 16) + 2].var, (arr)[((gid) * 16) + 3].var, (arr)[((gid) * 16) + 4].var, (arr)[((gid) * 16) + 5].var, (arr)[((gid) * 16) + 6].var, (arr)[((gid) * 16) + 7].var, (arr)[((gid) * 16) + 8].var, (arr)[((gid) * 16) + 9].var, (arr)[((gid) * 16) + 10].var, (arr)[((gid) * 16) + 11].var, (arr)[((gid) * 16) + 12].var, (arr)[((gid) * 16) + 13].var, (arr)[((gid) * 16) + 14].var, (arr)[((gid) * 16) + 15].var) -#endif - -#if VECT_SIZE == 1 -#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 1) + 0].var[(idx)]) -#elif VECT_SIZE == 2 -#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 2) + 0].var[(idx)], (arr)[((gid) * 2) + 1].var[(idx)]) -#elif VECT_SIZE == 4 -#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 4) + 0].var[(idx)], (arr)[((gid) * 4) + 1].var[(idx)], (arr)[((gid) * 4) + 2].var[(idx)], (arr)[((gid) * 4) + 3].var[(idx)]) -#elif VECT_SIZE == 8 -#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 8) + 0].var[(idx)], (arr)[((gid) * 8) + 1].var[(idx)], (arr)[((gid) * 8) + 2].var[(idx)], (arr)[((gid) * 8) + 3].var[(idx)], (arr)[((gid) * 8) + 4].var[(idx)], (arr)[((gid) * 8) + 5].var[(idx)], (arr)[((gid) * 8) + 6].var[(idx)], (arr)[((gid) * 8) + 7].var[(idx)]) -#elif VECT_SIZE == 16 -#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 16) + 0].var[(idx)], (arr)[((gid) * 16) + 1].var[(idx)], (arr)[((gid) * 16) + 2].var[(idx)], (arr)[((gid) * 16) + 3].var[(idx)], (arr)[((gid) * 16) + 4].var[(idx)], (arr)[((gid) * 16) + 5].var[(idx)], (arr)[((gid) * 16) + 6].var[(idx)], (arr)[((gid) * 16) + 7].var[(idx)], (arr)[((gid) * 16) + 8].var[(idx)], (arr)[((gid) * 16) + 9].var[(idx)], (arr)[((gid) * 16) + 10].var[(idx)], (arr)[((gid) * 16) + 11].var[(idx)], (arr)[((gid) * 16) + 12].var[(idx)], (arr)[((gid) * 16) + 13].var[(idx)], (arr)[((gid) * 16) + 14].var[(idx)], (arr)[((gid) * 16) + 15].var[(idx)]) -#endif - -#if VECT_SIZE == 1 -#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 1) + 0].var) -#elif VECT_SIZE == 2 -#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 2) + 0].var, (arr)[((gid) * 2) + 1].var) -#elif VECT_SIZE == 4 -#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 4) + 0].var, (arr)[((gid) * 4) + 1].var, (arr)[((gid) * 4) + 2].var, (arr)[((gid) * 4) + 3].var) -#elif VECT_SIZE == 8 -#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 8) + 0].var, (arr)[((gid) * 8) + 1].var, (arr)[((gid) * 8) + 2].var, (arr)[((gid) * 8) + 3].var, (arr)[((gid) * 8) + 4].var, (arr)[((gid) * 8) + 5].var, (arr)[((gid) * 8) + 6].var, (arr)[((gid) * 8) + 7].var) -#elif VECT_SIZE == 16 -#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 16) + 0].var, (arr)[((gid) * 16) + 1].var, (arr)[((gid) * 16) + 2].var, (arr)[((gid) * 16) + 3].var, (arr)[((gid) * 16) + 4].var, (arr)[((gid) * 16) + 5].var, (arr)[((gid) * 16) + 6].var, (arr)[((gid) * 16) + 7].var, (arr)[((gid) * 16) + 8].var, (arr)[((gid) * 16) + 9].var, (arr)[((gid) * 16) + 10].var, (arr)[((gid) * 16) + 11].var, (arr)[((gid) * 16) + 12].var, (arr)[((gid) * 16) + 13].var, (arr)[((gid) * 16) + 14].var, (arr)[((gid) * 16) + 15].var) -#endif - -#if VECT_SIZE == 1 -#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 1) + 0].var[(idx)] = val; -#elif VECT_SIZE == 2 -#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 2) + 0].var[(idx)] = val.s0; (arr)[((gid) * 2) + 1].var[(idx)] = val.s1; -#elif VECT_SIZE == 4 -#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 4) + 0].var[(idx)] = val.s0; (arr)[((gid) * 4) + 1].var[(idx)] = val.s1; (arr)[((gid) * 4) + 2].var[(idx)] = val.s2; (arr)[((gid) * 4) + 3].var[(idx)] = val.s3; -#elif VECT_SIZE == 8 -#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 8) + 0].var[(idx)] = val.s0; (arr)[((gid) * 8) + 1].var[(idx)] = val.s1; (arr)[((gid) * 8) + 2].var[(idx)] = val.s2; (arr)[((gid) * 8) + 3].var[(idx)] = val.s3; (arr)[((gid) * 8) + 4].var[(idx)] = val.s4; (arr)[((gid) * 8) + 5].var[(idx)] = val.s5; (arr)[((gid) * 8) + 6].var[(idx)] = val.s6; (arr)[((gid) * 8) + 7].var[(idx)] = val.s7; -#elif VECT_SIZE == 16 -#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 16) + 0].var[(idx)] = val.s0; (arr)[((gid) * 16) + 1].var[(idx)] = val.s1; (arr)[((gid) * 16) + 2].var[(idx)] = val.s2; (arr)[((gid) * 16) + 3].var[(idx)] = val.s3; (arr)[((gid) * 16) + 4].var[(idx)] = val.s4; (arr)[((gid) * 16) + 5].var[(idx)] = val.s5; (arr)[((gid) * 16) + 6].var[(idx)] = val.s6; (arr)[((gid) * 16) + 7].var[(idx)] = val.s7; (arr)[((gid) * 16) + 8].var[(idx)] = val.s8; (arr)[((gid) * 16) + 9].var[(idx)] = val.s9; (arr)[((gid) * 16) + 10].var[(idx)] = val.sa; (arr)[((gid) * 16) + 11].var[(idx)] = val.sb; (arr)[((gid) * 16) + 12].var[(idx)] = val.sc; (arr)[((gid) * 16) + 13].var[(idx)] = val.sd; (arr)[((gid) * 16) + 14].var[(idx)] = val.se; (arr)[((gid) * 16) + 15].var[(idx)] = val.sf; -#endif diff --git a/OpenCL/types_ocl.c b/OpenCL/types_ocl.c deleted file mode 100644 index 1d3a7b8..0000000 --- a/OpenCL/types_ocl.c +++ /dev/null @@ -1,1480 +0,0 @@ -/** - * Authors.....: Jens Steube - * magnum - * - * License.....: MIT - */ - -typedef uchar u8; -typedef ushort u16; -typedef uint u32; -typedef ulong u64; - -#ifndef NEW_SIMD_CODE -#undef VECT_SIZE -#define VECT_SIZE 1 -#endif - -#define CONCAT(a, b) a##b -#define VTYPE(type, width) CONCAT(type, width) - -#if VECT_SIZE == 1 -typedef uchar u8x; -typedef ushort u16x; -typedef uint u32x; -typedef ulong u64x; -#else -typedef VTYPE(uchar, VECT_SIZE) u8x; -typedef VTYPE(ushort, VECT_SIZE) u16x; -typedef VTYPE(uint, VECT_SIZE) u32x; -typedef VTYPE(ulong, VECT_SIZE) u64x; -#endif - -inline u32 l32_from_64_S (u64 a) -{ - const u32 r = (u32) (a); - - return r; -} - -inline u32 h32_from_64_S (u64 a) -{ - a >>= 32; - - const u32 r = (u32) (a); - - return r; -} - -inline u64 hl32_to_64_S (const u32 a, const u32 b) -{ - return as_ulong ((uint2) (b, a)); -} - -inline u32x l32_from_64 (u64x a) -{ - u32x r; - - #if VECT_SIZE == 1 - r = (u32) a; - #endif - - #if VECT_SIZE >= 2 - r.s0 = (u32) a.s0; - r.s1 = (u32) a.s1; - #endif - - #if VECT_SIZE >= 4 - r.s2 = (u32) a.s2; - r.s3 = (u32) a.s3; - #endif - - #if VECT_SIZE >= 8 - r.s4 = (u32) a.s4; - r.s5 = (u32) a.s5; - r.s6 = (u32) a.s6; - r.s7 = (u32) a.s7; - #endif - - #if VECT_SIZE >= 16 - r.s8 = (u32) a.s8; - r.s9 = (u32) a.s9; - r.sa = (u32) a.sa; - r.sb = (u32) a.sb; - r.sc = (u32) a.sc; - r.sd = (u32) a.sd; - r.se = (u32) a.se; - r.sf = (u32) a.sf; - #endif - - return r; -} - -inline u32x h32_from_64 (u64x a) -{ - a >>= 32; - - u32x r; - - #if VECT_SIZE == 1 - r = (u32) a; - #endif - - #if VECT_SIZE >= 2 - r.s0 = (u32) a.s0; - r.s1 = (u32) a.s1; - #endif - - #if VECT_SIZE >= 4 - r.s2 = (u32) a.s2; - r.s3 = (u32) a.s3; - #endif - - #if VECT_SIZE >= 8 - r.s4 = (u32) a.s4; - r.s5 = (u32) a.s5; - r.s6 = (u32) a.s6; - r.s7 = (u32) a.s7; - #endif - - #if VECT_SIZE >= 16 - r.s8 = (u32) a.s8; - r.s9 = (u32) a.s9; - r.sa = (u32) a.sa; - r.sb = (u32) a.sb; - r.sc = (u32) a.sc; - r.sd = (u32) a.sd; - r.se = (u32) a.se; - r.sf = (u32) a.sf; - #endif - - return r; -} - -inline u64x hl32_to_64 (const u32x a, const u32x b) -{ - u64x r; - - #if VECT_SIZE == 1 - r = as_ulong ((uint2) (b, a)); - #endif - - #if VECT_SIZE >= 2 - r.s0 = as_ulong ((uint2) (b.s0, a.s0)); - r.s1 = as_ulong ((uint2) (b.s1, a.s1)); - #endif - - #if VECT_SIZE >= 4 - r.s2 = as_ulong ((uint2) (b.s2, a.s2)); - r.s3 = as_ulong ((uint2) (b.s3, a.s3)); - #endif - - #if VECT_SIZE >= 8 - r.s4 = as_ulong ((uint2) (b.s4, a.s4)); - r.s5 = as_ulong ((uint2) (b.s5, a.s5)); - r.s6 = as_ulong ((uint2) (b.s6, a.s6)); - r.s7 = as_ulong ((uint2) (b.s7, a.s7)); - #endif - - #if VECT_SIZE >= 16 - r.s8 = as_ulong ((uint2) (b.s8, a.s8)); - r.s9 = as_ulong ((uint2) (b.s9, a.s9)); - r.sa = as_ulong ((uint2) (b.sa, a.sa)); - r.sb = as_ulong ((uint2) (b.sb, a.sb)); - r.sc = as_ulong ((uint2) (b.sc, a.sc)); - r.sd = as_ulong ((uint2) (b.sd, a.sd)); - r.se = as_ulong ((uint2) (b.se, a.se)); - r.sf = as_ulong ((uint2) (b.sf, a.sf)); - #endif - - return r; -} - -#ifdef IS_AMD -inline u32 swap32_S (const u32 v) -{ - return (as_uint (as_uchar4 (v).s3210)); -} - -inline u64 swap64_S (const u64 v) -{ - return (as_ulong (as_uchar8 (v).s76543210)); -} - -inline u32 rotr32_S (const u32 a, const u32 n) -{ - return rotate (a, 32 - n); -} - -inline u32 rotl32_S (const u32 a, const u32 n) -{ - return rotate (a, n); -} - -inline u64 rotr64_S (const u64 a, const u32 n) -{ - const u32 a0 = h32_from_64_S (a); - const u32 a1 = l32_from_64_S (a); - - const u32 t0 = (n >= 32) ? amd_bitalign (a0, a1, n - 32) : amd_bitalign (a1, a0, n); - const u32 t1 = (n >= 32) ? amd_bitalign (a1, a0, n - 32) : amd_bitalign (a0, a1, n); - - const u64 r = hl32_to_64_S (t0, t1); - - return r; -} - -inline u64 rotl64_S (const u64 a, const u32 n) -{ - return rotr64_S (a, 64 - n); -} - -inline u32x swap32 (const u32x v) -{ - return ((v >> 24) & 0x000000ff) - | ((v >> 8) & 0x0000ff00) - | ((v << 8) & 0x00ff0000) - | ((v << 24) & 0xff000000); -} - -inline u64x swap64 (const u64x v) -{ - return ((v >> 56) & 0x00000000000000ff) - | ((v >> 40) & 0x000000000000ff00) - | ((v >> 24) & 0x0000000000ff0000) - | ((v >> 8) & 0x00000000ff000000) - | ((v << 8) & 0x000000ff00000000) - | ((v << 24) & 0x0000ff0000000000) - | ((v << 40) & 0x00ff000000000000) - | ((v << 56) & 0xff00000000000000); -} - -inline u32x rotr32 (const u32x a, const u32 n) -{ - return rotate (a, 32 - n); -} - -inline u32x rotl32 (const u32x a, const u32 n) -{ - return rotate (a, n); -} - -inline u64x rotr64 (const u64x a, const u32 n) -{ - const u32x a0 = h32_from_64 (a); - const u32x a1 = l32_from_64 (a); - - const u32x t0 = (n >= 32) ? amd_bitalign (a0, a1, n - 32) : amd_bitalign (a1, a0, n); - const u32x t1 = (n >= 32) ? amd_bitalign (a1, a0, n - 32) : amd_bitalign (a0, a1, n); - - const u64x r = hl32_to_64 (t0, t1); - - return r; -} - -inline u64x rotl64 (const u64x a, const u32 n) -{ - return rotr64 (a, 64 - n); -} - -inline u32x __bfe (const u32x a, const u32x b, const u32x c) -{ - return amd_bfe (a, b, c); -} - -inline u32 __bfe_S (const u32 a, const u32 b, const u32 c) -{ - return amd_bfe (a, b, c); -} - -inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c) -{ - return amd_bytealign (a, b, c); -} -#endif - -#ifdef IS_NV -inline u32 swap32_S (const u32 v) -{ - u32 r; - - asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(r) : "r"(v)); - - return r; -} - -inline u64 swap64_S (const u64 v) -{ - u32 il; - u32 ir; - - asm ("mov.b64 {%0, %1}, %2;" : "=r"(il), "=r"(ir) : "l"(v)); - - u32 tl; - u32 tr; - - asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(tl) : "r"(il)); - asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(tr) : "r"(ir)); - - u64 r; - - asm ("mov.b64 %0, {%1, %2};" : "=l"(r) : "r"(tr), "r"(tl)); - - return r; -} - -inline u32 rotr32_S (const u32 a, const u32 n) -{ - return rotate (a, 32 - n); -} - -inline u32 rotl32_S (const u32 a, const u32 n) -{ - return rotate (a, n); -} - -inline u64 rotr64_S (const u64 a, const u32 n) -{ - return rotate (a, (u64) 64 - n); -} - -inline u64 rotl64_S (const u64 a, const u32 n) -{ - return rotr64_S (a, 64 - n); -} - -inline u32x swap32 (const u32x v) -{ - return ((v >> 24) & 0x000000ff) - | ((v >> 8) & 0x0000ff00) - | ((v << 8) & 0x00ff0000) - | ((v << 24) & 0xff000000); -} - -inline u64x swap64 (const u64x v) -{ - return ((v >> 56) & 0x00000000000000ff) - | ((v >> 40) & 0x000000000000ff00) - | ((v >> 24) & 0x0000000000ff0000) - | ((v >> 8) & 0x00000000ff000000) - | ((v << 8) & 0x000000ff00000000) - | ((v << 24) & 0x0000ff0000000000) - | ((v << 40) & 0x00ff000000000000) - | ((v << 56) & 0xff00000000000000); -} - -inline u32x rotr32 (const u32x a, const u32 n) -{ - return rotate (a, 32 - n); -} - -inline u32x rotl32 (const u32x a, const u32 n) -{ - return rotate (a, n); -} - -inline u64x rotr64 (const u64x a, const u32 n) -{ - return rotate (a, (u64) 64 - n); -} - -inline u64x rotl64 (const u64x a, const u32 n) -{ - return rotate (a, (u64) n); -} - -inline u32x __byte_perm (const u32x a, const u32x b, const u32x c) -{ - u32x r; - - #if VECT_SIZE == 1 - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c) ); - #endif - - #if VECT_SIZE >= 2 - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(a.s0), "r"(b.s0), "r"(c.s0)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(a.s1), "r"(b.s1), "r"(c.s1)); - #endif - - #if VECT_SIZE >= 4 - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(a.s2), "r"(b.s2), "r"(c.s2)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(a.s3), "r"(b.s3), "r"(c.s3)); - #endif - - #if VECT_SIZE >= 8 - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(a.s4), "r"(b.s4), "r"(c.s4)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(a.s5), "r"(b.s5), "r"(c.s5)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(a.s6), "r"(b.s6), "r"(c.s6)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(a.s7), "r"(b.s7), "r"(c.s7)); - #endif - - #if VECT_SIZE >= 16 - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(a.s8), "r"(b.s8), "r"(c.s8)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(a.s9), "r"(b.s9), "r"(c.s9)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(a.sa), "r"(b.sa), "r"(c.sa)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(a.sb), "r"(b.sb), "r"(c.sb)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(a.sc), "r"(b.sc), "r"(c.sc)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(a.sd), "r"(b.sd), "r"(c.sd)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(a.se), "r"(b.se), "r"(c.se)); - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(a.sf), "r"(b.sf), "r"(c.sf)); - #endif - - return r; -} - -inline u32 __byte_perm_S (const u32 a, const u32 b, const u32 c) -{ - u32 r; - - asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); - - return r; -} - -inline u32x __bfe (const u32x a, const u32x b, const u32x c) -{ - u32x r; - - #if VECT_SIZE == 1 - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); - #endif - - #if VECT_SIZE >= 2 - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(a.s0), "r"(b.s0), "r"(c.s0)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(a.s1), "r"(b.s1), "r"(c.s1)); - #endif - - #if VECT_SIZE >= 4 - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(a.s2), "r"(b.s2), "r"(c.s2)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(a.s3), "r"(b.s3), "r"(c.s3)); - #endif - - #if VECT_SIZE >= 8 - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(a.s4), "r"(b.s4), "r"(c.s4)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(a.s5), "r"(b.s5), "r"(c.s5)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(a.s6), "r"(b.s6), "r"(c.s6)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(a.s7), "r"(b.s7), "r"(c.s7)); - #endif - - #if VECT_SIZE >= 16 - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(a.s8), "r"(b.s8), "r"(c.s8)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(a.s9), "r"(b.s9), "r"(c.s9)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(a.sa), "r"(b.sa), "r"(c.sa)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(a.sb), "r"(b.sb), "r"(c.sb)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(a.sc), "r"(b.sc), "r"(c.sc)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(a.sd), "r"(b.sd), "r"(c.sd)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(a.se), "r"(b.se), "r"(c.se)); - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(a.sf), "r"(b.sf), "r"(c.sf)); - #endif - - return r; -} - -inline u32 __bfe_S (const u32 a, const u32 b, const u32 c) -{ - u32 r; - - asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); - - return r; -} - -inline u32x amd_bytealign (const u32x a, const u32x b, const u32x c) -{ - u32x r; - - #if CUDA_ARCH >= 350 - - #if VECT_SIZE == 1 - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(b), "r"(a), "r"((c & 3) * 8)); - #endif - - #if VECT_SIZE >= 2 - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(b.s0), "r"(a.s0), "r"((c.s0 & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(b.s1), "r"(a.s1), "r"((c.s1 & 3) * 8)); - #endif - - #if VECT_SIZE >= 4 - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(b.s2), "r"(a.s2), "r"((c.s2 & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(b.s3), "r"(a.s3), "r"((c.s3 & 3) * 8)); - #endif - - #if VECT_SIZE >= 8 - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(b.s4), "r"(a.s4), "r"((c.s4 & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(b.s5), "r"(a.s5), "r"((c.s5 & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(b.s6), "r"(a.s6), "r"((c.s6 & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(b.s7), "r"(a.s7), "r"((c.s7 & 3) * 8)); - #endif - - #if VECT_SIZE >= 16 - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(b.s8), "r"(a.s8), "r"((c.s8 & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(b.s9), "r"(a.s9), "r"((c.s9 & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(b.sa), "r"(a.sa), "r"((c.sa & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(b.sb), "r"(a.sb), "r"((c.sb & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(b.sc), "r"(a.sc), "r"((c.sc & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(b.sd), "r"(a.sd), "r"((c.sd & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(b.se), "r"(a.se), "r"((c.se & 3) * 8)); - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(b.sf), "r"(a.sf), "r"((c.sf & 3) * 8)); - #endif - - #else - - r = __byte_perm (b, a, ((u32x) (0x76543210) >> ((c & 3) * 4)) & 0xffff); - - #endif - - return r; -} - -inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c) -{ - u32 r; - - #if CUDA_ARCH >= 350 - - asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(b), "r"(a), "r"((c & 3) * 8)); - - #else - - r = __byte_perm_S (b, a, (0x76543210 >> ((c & 3) * 4)) & 0xffff); - - #endif - - return r; -} -#endif - -#ifdef IS_GENERIC -inline u32 swap32_S (const u32 v) -{ - return (as_uint (as_uchar4 (v).s3210)); -} - -inline u64 swap64_S (const u64 v) -{ - return (as_ulong (as_uchar8 (v).s76543210)); -} - -inline u32 rotr32_S (const u32 a, const u32 n) -{ - return rotate (a, 32 - n); -} - -inline u32 rotl32_S (const u32 a, const u32 n) -{ - return rotate (a, n); -} - -inline u64 rotr64_S (const u64 a, const u32 n) -{ - return rotate (a, (u64) 64 - n); -} - -inline u64 rotl64_S (const u64 a, const u32 n) -{ - return rotate (a, (u64) n); -} - -inline u32x swap32 (const u32x v) -{ - return ((v >> 24) & 0x000000ff) - | ((v >> 8) & 0x0000ff00) - | ((v << 8) & 0x00ff0000) - | ((v << 24) & 0xff000000); -} - -inline u64x swap64 (const u64x v) -{ - return ((v >> 56) & 0x00000000000000ff) - | ((v >> 40) & 0x000000000000ff00) - | ((v >> 24) & 0x0000000000ff0000) - | ((v >> 8) & 0x00000000ff000000) - | ((v << 8) & 0x000000ff00000000) - | ((v << 24) & 0x0000ff0000000000) - | ((v << 40) & 0x00ff000000000000) - | ((v << 56) & 0xff00000000000000); -} - -inline u32x rotr32 (const u32x a, const u32 n) -{ - return rotate (a, 32 - n); -} - -inline u32x rotl32 (const u32x a, const u32 n) -{ - return rotate (a, n); -} - -inline u64x rotr64 (const u64x a, const u32 n) -{ - return rotate (a, (u64) 64 - n); -} - -inline u64x rotl64 (const u64x a, const u32 n) -{ - return rotate (a, (u64) n); -} - -inline u32x __bfe (const u32x a, const u32x b, const u32x c) -{ - #define BIT(x) ((u32x) (1u) << (x)) - #define BIT_MASK(x) (BIT (x) - 1) - #define BFE(x,y,z) (((x) >> (y)) & BIT_MASK (z)) - - return BFE (a, b, c); - - #undef BIT - #undef BIT_MASK - #undef BFE -} - -inline u32 __bfe_S (const u32 a, const u32 b, const u32 c) -{ - #define BIT(x) (1u << (x)) - #define BIT_MASK(x) (BIT (x) - 1) - #define BFE(x,y,z) (((x) >> (y)) & BIT_MASK (z)) - - return BFE (a, b, c); - - #undef BIT - #undef BIT_MASK - #undef BFE -} - -inline u32x amd_bytealign (const u32x a, const u32x b, const u32 c) -{ - #if VECT_SIZE == 1 - const u64x tmp = ((((u64x) (a)) << 32) | ((u64x) (b))) >> ((c & 3) * 8); - - return (u32x) (tmp); - #endif - - #if VECT_SIZE == 2 - const u64x tmp = ((((u64x) (a.s0, a.s1)) << 32) | ((u64x) (b.s0, b.s1))) >> ((c & 3) * 8); - - return (u32x) (tmp.s0, tmp.s1); - #endif - - #if VECT_SIZE == 4 - const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3))) >> ((c & 3) * 8); - - return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3); - #endif - - #if VECT_SIZE == 8 - const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3, a.s4, a.s5, a.s6, a.s7)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3, b.s4, b.s5, b.s6, b.s7))) >> ((c & 3) * 8); - - return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3, tmp.s4, tmp.s5, tmp.s6, tmp.s7); - #endif - - #if VECT_SIZE == 16 - const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3, a.s4, a.s5, a.s6, a.s7, a.s8, a.s9, a.sa, a.sb, a.sc, a.sd, a.se, a.sf)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3, b.s4, b.s5, b.s6, b.s7, b.s8, b.s9, b.sa, b.sb, b.sc, b.sd, b.se, b.sf))) >> ((c & 3) * 8); - - return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3, tmp.s4, tmp.s5, tmp.s6, tmp.s7, tmp.s8, tmp.s9, tmp.sa, tmp.sb, tmp.sc, tmp.sd, tmp.se, tmp.sf); - #endif -} - -inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c) -{ - const u64 tmp = ((((u64) a) << 32) | ((u64) b)) >> ((c & 3) * 8); - - return (u32) (tmp); -} - -#endif - -typedef struct -{ - #if defined _DES_ - u32 digest_buf[4]; - #elif defined _MD4_ - u32 digest_buf[4]; - #elif defined _MD5_ - u32 digest_buf[4]; - #elif defined _MD5H_ - u32 digest_buf[4]; - #elif defined _SHA1_ - u32 digest_buf[5]; - #elif defined _BCRYPT_ - u32 digest_buf[6]; - #elif defined _SHA256_ - u32 digest_buf[8]; - #elif defined _SHA384_ - u32 digest_buf[16]; - #elif defined _SHA512_ - u32 digest_buf[16]; - #elif defined _KECCAK_ - u32 digest_buf[50]; - #elif defined _RIPEMD160_ - u32 digest_buf[5]; - #elif defined _WHIRLPOOL_ - u32 digest_buf[16]; - #elif defined _GOST_ - u32 digest_buf[8]; - #elif defined _GOST2012_256_ - u32 digest_buf[8]; - #elif defined _GOST2012_512_ - u32 digest_buf[16]; - #elif defined _SAPB_ - u32 digest_buf[4]; - #elif defined _SAPG_ - u32 digest_buf[5]; - #elif defined _MYSQL323_ - u32 digest_buf[4]; - #elif defined _LOTUS5_ - u32 digest_buf[4]; - #elif defined _LOTUS6_ - u32 digest_buf[4]; - #elif defined _SCRYPT_ - u32 digest_buf[8]; - #elif defined _LOTUS8_ - u32 digest_buf[4]; - #elif defined _OFFICE2007_ - u32 digest_buf[4]; - #elif defined _OFFICE2010_ - u32 digest_buf[4]; - #elif defined _OFFICE2013_ - u32 digest_buf[4]; - #elif defined _OLDOFFICE01_ - u32 digest_buf[4]; - #elif defined _OLDOFFICE34_ - u32 digest_buf[4]; - #elif defined _SIPHASH_ - u32 digest_buf[4]; - #elif defined _PBKDF2_MD5_ - u32 digest_buf[32]; - #elif defined _PBKDF2_SHA1_ - u32 digest_buf[32]; - #elif defined _PBKDF2_SHA256_ - u32 digest_buf[32]; - #elif defined _PBKDF2_SHA512_ - u32 digest_buf[32]; - #elif defined _PDF17L8_ - u32 digest_buf[8]; - #elif defined _CRC32_ - u32 digest_buf[4]; - #elif defined _SEVEN_ZIP_ - u32 digest_buf[4]; - #elif defined _ANDROIDFDE_ - u32 digest_buf[4]; - #elif defined _DCC2_ - u32 digest_buf[4]; - #elif defined _WPA_ - u32 digest_buf[4]; - #elif defined _MD5_SHA1_ - u32 digest_buf[4]; - #elif defined _SHA1_MD5_ - u32 digest_buf[5]; - #elif defined _NETNTLMV2_ - u32 digest_buf[4]; - #elif defined _KRB5PA_ - u32 digest_buf[4]; - #elif defined _CLOUDKEY_ - u32 digest_buf[8]; - #elif defined _SCRYPT_ - u32 digest_buf[4]; - #elif defined _PSAFE2_ - u32 digest_buf[5]; - #elif defined _LOTUS8_ - u32 digest_buf[4]; - #elif defined _RAR3_ - u32 digest_buf[4]; - #elif defined _SHA256_SHA1_ - u32 digest_buf[8]; - #elif defined _MS_DRSR_ - u32 digest_buf[8]; - #elif defined _ANDROIDFDE_SAMSUNG_ - u32 digest_buf[8]; - #elif defined _RAR5_ - u32 digest_buf[4]; - #elif defined _KRB5TGS_ - u32 digest_buf[4]; - #elif defined _AXCRYPT_ - u32 digest_buf[4]; - #elif defined _KEEPASS_ - u32 digest_buf[4]; - #elif defined _ZIP2_ - u32 digest_buf[4]; - #endif - -} digest_t; - -typedef struct -{ - u32 salt_buf[16]; - u32 salt_buf_pc[8]; - - u32 salt_len; - u32 salt_iter; - u32 salt_sign[2]; - - u32 keccak_mdlen; - u32 truecrypt_mdlen; - - u32 digests_cnt; - u32 digests_done; - - u32 digests_offset; - - u32 scrypt_N; - u32 scrypt_r; - u32 scrypt_p; - u32 scrypt_tmto; - u32 scrypt_phy; - -} salt_t; - -typedef struct -{ - int V; - int R; - int P; - - int enc_md; - - u32 id_buf[8]; - u32 u_buf[32]; - u32 o_buf[32]; - - int id_len; - int o_len; - int u_len; - - u32 rc4key[2]; - u32 rc4data[2]; - -} pdf_t; - -typedef struct -{ - u32 pke[25]; - u32 eapol[64]; - int eapol_size; - int keyver; - u8 orig_mac1[6]; - u8 orig_mac2[6]; - u8 orig_nonce1[32]; - u8 orig_nonce2[32]; - -} wpa_t; - -typedef struct -{ - u32 cry_master_buf[64]; - u32 ckey_buf[64]; - u32 public_key_buf[64]; - - u32 cry_master_len; - u32 ckey_len; - u32 public_key_len; - -} bitcoin_wallet_t; - -typedef struct -{ - u32 salt_buf[30]; - u32 salt_len; - - u32 esalt_buf[38]; - u32 esalt_len; - -} sip_t; - -typedef struct -{ - u32 data[384]; - -} androidfde_t; - -typedef struct -{ - u32 nr_buf[16]; - u32 nr_len; - - u32 msg_buf[128]; - u32 msg_len; - -} ikepsk_t; - -typedef struct -{ - u32 user_len; - u32 domain_len; - u32 srvchall_len; - u32 clichall_len; - - u32 userdomain_buf[64]; - u32 chall_buf[256]; - -} netntlm_t; - -typedef struct -{ - u32 user[16]; - u32 realm[16]; - u32 salt[32]; - u32 timestamp[16]; - u32 checksum[4]; - -} krb5pa_t; - -typedef struct -{ - u32 account_info[512]; - u32 checksum[4]; - u32 edata2[2560]; - u32 edata2_len; - -} krb5tgs_t; - -typedef struct -{ - u32 salt_buf[16]; - u32 data_buf[112]; - u32 keyfile_buf[16]; - u32 signature; - -} tc_t; - -typedef struct -{ - u32 salt_buf[16]; - -} pbkdf2_md5_t; - -typedef struct -{ - u32 salt_buf[16]; - -} pbkdf2_sha1_t; - -typedef struct -{ - u32 salt_buf[16]; - -} pbkdf2_sha256_t; - -typedef struct -{ - u32 salt_buf[32]; - -} pbkdf2_sha512_t; - -typedef struct -{ - u32 salt_buf[128]; - u32 salt_len; - -} rakp_t; - -typedef struct -{ - u32 data_len; - u32 data_buf[512]; - -} cloudkey_t; - -typedef struct -{ - u32 encryptedVerifier[4]; - u32 encryptedVerifierHash[5]; - - u32 keySize; - -} office2007_t; - -typedef struct -{ - u32 encryptedVerifier[4]; - u32 encryptedVerifierHash[8]; - -} office2010_t; - -typedef struct -{ - u32 encryptedVerifier[4]; - u32 encryptedVerifierHash[8]; - -} office2013_t; - -typedef struct -{ - u32 version; - u32 encryptedVerifier[4]; - u32 encryptedVerifierHash[4]; - u32 rc4key[2]; - -} oldoffice01_t; - -typedef struct -{ - u32 version; - u32 encryptedVerifier[4]; - u32 encryptedVerifierHash[5]; - u32 rc4key[2]; - -} oldoffice34_t; - -typedef struct -{ - u32 salt_buf[128]; - u32 salt_len; - - u32 pc_digest[5]; - u32 pc_offset; - -} pstoken_t; - -typedef struct -{ - u32 type; - u32 mode; - u32 magic; - u32 salt_len; - u32 salt_buf[4]; - u32 verify_bytes; - u32 compress_length; - u32 data_len; - u32 data_buf[2048]; - u32 auth_len; - u32 auth_buf[4]; - -} zip2_t; - -typedef struct -{ - u32 version; - u32 algorithm; - - /* key-file handling */ - u32 keyfile_len; - u32 keyfile[8]; - - u32 final_random_seed[8]; - u32 transf_random_seed[8]; - u32 enc_iv[4]; - u32 contents_hash[8]; - - /* specific to version 1 */ - u32 contents_len; - u32 contents[75000]; - - /* specific to version 2 */ - u32 expected_bytes[8]; - -} keepass_t; - -typedef struct -{ - u32 digest[4]; - u32 out[4]; - -} pdf14_tmp_t; - -typedef struct -{ - union - { - u32 dgst32[16]; - u64 dgst64[8]; - }; - - u32 dgst_len; - u32 W_len; - -} pdf17l8_tmp_t; - -typedef struct -{ - u32 digest_buf[4]; - -} phpass_tmp_t; - -typedef struct -{ - u32 digest_buf[4]; - -} md5crypt_tmp_t; - -typedef struct -{ - u32 alt_result[8]; - - u32 p_bytes[4]; - u32 s_bytes[4]; - -} sha256crypt_tmp_t; - -typedef struct -{ - u64 l_alt_result[8]; - - u64 l_p_bytes[2]; - u64 l_s_bytes[2]; - -} sha512crypt_tmp_t; - -typedef struct -{ - u32 ipad[5]; - u32 opad[5]; - - u32 dgst[10]; - u32 out[10]; - -} wpa_tmp_t; - -typedef struct -{ - u64 dgst[8]; - -} bitcoin_wallet_tmp_t; - -typedef struct -{ - u32 ipad[5]; - u32 opad[5]; - - u32 dgst[5]; - u32 out[4]; - -} dcc2_tmp_t; - -typedef struct -{ - u32 E[18]; - - u32 P[18]; - - u32 S0[256]; - u32 S1[256]; - u32 S2[256]; - u32 S3[256]; - -} bcrypt_tmp_t; - -typedef struct -{ - u32 digest[2]; - - u32 P[18]; - - u32 S0[256]; - u32 S1[256]; - u32 S2[256]; - u32 S3[256]; - -} pwsafe2_tmp_t; - -typedef struct -{ - u32 digest_buf[8]; - -} pwsafe3_tmp_t; - -typedef struct -{ - u32 digest_buf[5]; - -} androidpin_tmp_t; - -typedef struct -{ - u32 ipad[5]; - u32 opad[5]; - - u32 dgst[10]; - u32 out[10]; - -} androidfde_tmp_t; - -typedef struct -{ - u32 ipad[16]; - u32 opad[16]; - - u32 dgst[64]; - u32 out[64]; - -} tc_tmp_t; - -typedef struct -{ - u64 ipad[8]; - u64 opad[8]; - - u64 dgst[32]; - u64 out[32]; - -} tc64_tmp_t; - -typedef struct -{ - u32 ipad[4]; - u32 opad[4]; - - u32 dgst[32]; - u32 out[32]; - -} pbkdf2_md5_tmp_t; - -typedef struct -{ - u32 ipad[5]; - u32 opad[5]; - - u32 dgst[32]; - u32 out[32]; - -} pbkdf2_sha1_tmp_t; - -typedef struct -{ - u32 ipad[8]; - u32 opad[8]; - - u32 dgst[32]; - u32 out[32]; - -} pbkdf2_sha256_tmp_t; - -typedef struct -{ - u64 ipad[8]; - u64 opad[8]; - - u64 dgst[16]; - u64 out[16]; - -} pbkdf2_sha512_tmp_t; - -typedef struct -{ - u64 out[8]; - -} ecryptfs_tmp_t; - -typedef struct -{ - u64 ipad[8]; - u64 opad[8]; - - u64 dgst[16]; - u64 out[16]; - -} oraclet_tmp_t; - -typedef struct -{ - u32 ipad[5]; - u32 opad[5]; - - u32 dgst[5]; - u32 out[5]; - -} agilekey_tmp_t; - -typedef struct -{ - u32 ipad[5]; - u32 opad[5]; - - u32 dgst1[5]; - u32 out1[5]; - - u32 dgst2[5]; - u32 out2[5]; - -} mywallet_tmp_t; - -typedef struct -{ - u32 ipad[5]; - u32 opad[5]; - - u32 dgst[5]; - u32 out[5]; - -} sha1aix_tmp_t; - -typedef struct -{ - u32 ipad[8]; - u32 opad[8]; - - u32 dgst[8]; - u32 out[8]; - -} sha256aix_tmp_t; - -typedef struct -{ - u64 ipad[8]; - u64 opad[8]; - - u64 dgst[8]; - u64 out[8]; - -} sha512aix_tmp_t; - -typedef struct -{ - u32 ipad[8]; - u32 opad[8]; - - u32 dgst[8]; - u32 out[8]; - -} lastpass_tmp_t; - -typedef struct -{ - u64 digest_buf[8]; - -} drupal7_tmp_t; - -typedef struct -{ - u32 ipad[5]; - u32 opad[5]; - - u32 dgst[5]; - u32 out[5]; - -} lotus8_tmp_t; - -typedef struct -{ - u32 out[5]; - -} office2007_tmp_t; - -typedef struct -{ - u32 out[5]; - -} office2010_tmp_t; - -typedef struct -{ - u64 out[8]; - -} office2013_tmp_t; - -typedef struct -{ - u32 digest_buf[5]; - -} saph_sha1_tmp_t; - -typedef struct -{ - u32 block[16]; - - u32 dgst[8]; - - u32 block_len; - u32 final_len; - -} seven_zip_tmp_t; - -typedef struct -{ - u32 KEK[5]; - - u32 lsb[4]; - u32 cipher[4]; - -} axcrypt_tmp_t; - -typedef struct -{ - u32 tmp_digest[8]; - -} keepass_tmp_t; - -typedef struct -{ - u32 Kc[16]; - u32 Kd[16]; - - u32 iv[2]; - -} bsdicrypt_tmp_t; - -typedef struct -{ - u32 dgst[17][5]; - -} rar3_tmp_t; - -typedef struct -{ - u32 user[16]; - -} cram_md5_t; - -typedef struct -{ - u32 iv_buf[4]; - u32 iv_len; - - u32 salt_buf[4]; - u32 salt_len; - - u32 crc; - - u32 data_buf[96]; - u32 data_len; - - u32 unpack_size; - -} seven_zip_t; - -typedef struct -{ - u32 key; - u64 val; - -} hcstat_table_t; - -typedef struct -{ - u32 cs_buf[0x100]; - u32 cs_len; - -} cs_t; - -typedef struct -{ - u32 cmds[0x100]; - -} kernel_rule_t; - -typedef struct -{ - u32 salt_pos; - u32 digest_pos; - u32 hash_pos; - u32 gidvid; - u32 il_pos; - -} plain_t; - -typedef struct -{ - u32 i[16]; - - u32 pw_len; - - u32 alignment_placeholder_1; - u32 alignment_placeholder_2; - u32 alignment_placeholder_3; - -} pw_t; - -typedef struct -{ - u32 i; - -} bf_t; - -typedef struct -{ - u32 i[8]; - - u32 pw_len; - -} comb_t; - -typedef struct -{ - u32 b[32]; - -} bs_word_t; - -typedef struct -{ - uint4 P[64]; - -} scrypt_tmp_t; diff --git a/include/constants.h b/include/constants.h deleted file mode 100644 index 5351445..0000000 --- a/include/constants.h +++ /dev/null @@ -1,758 +0,0 @@ -/** - * Author......: Jens Steube - * License.....: MIT - */ - -#define COMBINATOR_MODE_BASE_LEFT 10001 -#define COMBINATOR_MODE_BASE_RIGHT 10002 - -#ifdef SHARED_H -#define _BCRYPT_ -#define _SHA1_ -#define _SHA256_ -#define _SHA384_ -#define _SHA512_ -#define _MD4_ -#define _MD5_ -#define _MD5H_ -#define _KECCAK_ -#define _RIPEMD160_ -#define _WHIRLPOOL_ -#define _SAPB_ -#define _CLOUDKEY_ -#define _OFFICE2013_ -#define _SIPHASH_ -#define _PDF17L8_ -#define _PBKDF2_MD5_ -#define _PBKDF2_SHA1_ -#define _PBKDF2_SHA256_ -#define _PBKDF2_SHA512_ -#define _RAR3_ -#define _ZIP2_ -#endif - -#ifdef _SIPHASH_ -/** - * SipHash Constants - */ - -#define SIPHASHM_0 0x736f6d6570736575 -#define SIPHASHM_1 0x646f72616e646f6d -#define SIPHASHM_2 0x6c7967656e657261 -#define SIPHASHM_3 0x7465646279746573 -#endif - -#if defined _BCRYPT_ || defined _PSAFE2_ -/** - * bcrypt Constants - */ - -#define BCRYPTM_0 0x4F727068u -#define BCRYPTM_1 0x65616E42u -#define BCRYPTM_2 0x65686F6Cu -#define BCRYPTM_3 0x64657253u -#define BCRYPTM_4 0x63727944u -#define BCRYPTM_5 0x6F756274u -#endif - -#if defined _SHA1_ || defined _SAPG_ || defined _OFFICE2007_ || defined _OFFICE2010_ || defined _OLDOFFICE34_ || defined _ANDROIDFDE_ || defined _DCC2_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _PSAFE2_ || defined _LOTUS8_ || defined _PBKDF2_SHA1_ || defined _RAR3_ || defined _SHA256_SHA1_ || defined _ZIP2_ -/** - * SHA1 Constants - */ - -#define SHA1M_A 0x67452301u -#define SHA1M_B 0xefcdab89u -#define SHA1M_C 0x98badcfeu -#define SHA1M_D 0x10325476u -#define SHA1M_E 0xc3d2e1f0u - -#define SHA1C00 0x5a827999u -#define SHA1C01 0x6ed9eba1u -#define SHA1C02 0x8f1bbcdcu -#define SHA1C03 0xca62c1d6u -#endif - -#if defined _SHA256_ || defined _PDF17L8_ || defined _SEVEN_ZIP_ || defined _ANDROIDFDE_ || defined _CLOUDKEY_ || defined _SCRYPT_ || defined _PBKDF2_SHA256_ || defined _SHA256_SHA1_ || defined _MS_DRSR_ || defined _ANDROIDFDE_SAMSUNG_ || defined _RAR5_ || defined _KEEPASS_ -/** - * SHA256 Constants - */ - -#define SHA256M_A 0x6a09e667u -#define SHA256M_B 0xbb67ae85u -#define SHA256M_C 0x3c6ef372u -#define SHA256M_D 0xa54ff53au -#define SHA256M_E 0x510e527fu -#define SHA256M_F 0x9b05688cu -#define SHA256M_G 0x1f83d9abu -#define SHA256M_H 0x5be0cd19u - -#define SHA256C00 0x428a2f98u -#define SHA256C01 0x71374491u -#define SHA256C02 0xb5c0fbcfu -#define SHA256C03 0xe9b5dba5u -#define SHA256C04 0x3956c25bu -#define SHA256C05 0x59f111f1u -#define SHA256C06 0x923f82a4u -#define SHA256C07 0xab1c5ed5u -#define SHA256C08 0xd807aa98u -#define SHA256C09 0x12835b01u -#define SHA256C0a 0x243185beu -#define SHA256C0b 0x550c7dc3u -#define SHA256C0c 0x72be5d74u -#define SHA256C0d 0x80deb1feu -#define SHA256C0e 0x9bdc06a7u -#define SHA256C0f 0xc19bf174u -#define SHA256C10 0xe49b69c1u -#define SHA256C11 0xefbe4786u -#define SHA256C12 0x0fc19dc6u -#define SHA256C13 0x240ca1ccu -#define SHA256C14 0x2de92c6fu -#define SHA256C15 0x4a7484aau -#define SHA256C16 0x5cb0a9dcu -#define SHA256C17 0x76f988dau -#define SHA256C18 0x983e5152u -#define SHA256C19 0xa831c66du -#define SHA256C1a 0xb00327c8u -#define SHA256C1b 0xbf597fc7u -#define SHA256C1c 0xc6e00bf3u -#define SHA256C1d 0xd5a79147u -#define SHA256C1e 0x06ca6351u -#define SHA256C1f 0x14292967u -#define SHA256C20 0x27b70a85u -#define SHA256C21 0x2e1b2138u -#define SHA256C22 0x4d2c6dfcu -#define SHA256C23 0x53380d13u -#define SHA256C24 0x650a7354u -#define SHA256C25 0x766a0abbu -#define SHA256C26 0x81c2c92eu -#define SHA256C27 0x92722c85u -#define SHA256C28 0xa2bfe8a1u -#define SHA256C29 0xa81a664bu -#define SHA256C2a 0xc24b8b70u -#define SHA256C2b 0xc76c51a3u -#define SHA256C2c 0xd192e819u -#define SHA256C2d 0xd6990624u -#define SHA256C2e 0xf40e3585u -#define SHA256C2f 0x106aa070u -#define SHA256C30 0x19a4c116u -#define SHA256C31 0x1e376c08u -#define SHA256C32 0x2748774cu -#define SHA256C33 0x34b0bcb5u -#define SHA256C34 0x391c0cb3u -#define SHA256C35 0x4ed8aa4au -#define SHA256C36 0x5b9cca4fu -#define SHA256C37 0x682e6ff3u -#define SHA256C38 0x748f82eeu -#define SHA256C39 0x78a5636fu -#define SHA256C3a 0x84c87814u -#define SHA256C3b 0x8cc70208u -#define SHA256C3c 0x90befffau -#define SHA256C3d 0xa4506cebu -#define SHA256C3e 0xbef9a3f7u -#define SHA256C3f 0xc67178f2u -#endif - -#if defined _MD4_ || defined _DCC2_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _MS_DRSR_ || defined _KRB5TGS_ -/** - * MD4 Constants - */ - -#define MD4M_A 0x67452301u -#define MD4M_B 0xefcdab89u -#define MD4M_C 0x98badcfeu -#define MD4M_D 0x10325476u - -#define MD4S00 3u -#define MD4S01 7u -#define MD4S02 11u -#define MD4S03 19u -#define MD4S10 3u -#define MD4S11 5u -#define MD4S12 9u -#define MD4S13 13u -#define MD4S20 3u -#define MD4S21 9u -#define MD4S22 11u -#define MD4S23 15u - -#define MD4C00 0x00000000u -#define MD4C01 0x5a827999u -#define MD4C02 0x6ed9eba1u -#endif - -#if defined _MD5_ || defined _MD5H_ || defined _SAPB_ || defined _OLDOFFICE01_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _PBKDF2_MD5_ || defined _KRB5TGS_ -/** - * MD5 Constants - */ - -#define MD5M_A 0x67452301u -#define MD5M_B 0xefcdab89u -#define MD5M_C 0x98badcfeu -#define MD5M_D 0x10325476u - -#define MD5S00 7u -#define MD5S01 12u -#define MD5S02 17u -#define MD5S03 22u -#define MD5S10 5u -#define MD5S11 9u -#define MD5S12 14u -#define MD5S13 20u -#define MD5S20 4u -#define MD5S21 11u -#define MD5S22 16u -#define MD5S23 23u -#define MD5S30 6u -#define MD5S31 10u -#define MD5S32 15u -#define MD5S33 21u - -#define MD5C00 0xd76aa478u -#define MD5C01 0xe8c7b756u -#define MD5C02 0x242070dbu -#define MD5C03 0xc1bdceeeu -#define MD5C04 0xf57c0fafu -#define MD5C05 0x4787c62au -#define MD5C06 0xa8304613u -#define MD5C07 0xfd469501u -#define MD5C08 0x698098d8u -#define MD5C09 0x8b44f7afu -#define MD5C0a 0xffff5bb1u -#define MD5C0b 0x895cd7beu -#define MD5C0c 0x6b901122u -#define MD5C0d 0xfd987193u -#define MD5C0e 0xa679438eu -#define MD5C0f 0x49b40821u -#define MD5C10 0xf61e2562u -#define MD5C11 0xc040b340u -#define MD5C12 0x265e5a51u -#define MD5C13 0xe9b6c7aau -#define MD5C14 0xd62f105du -#define MD5C15 0x02441453u -#define MD5C16 0xd8a1e681u -#define MD5C17 0xe7d3fbc8u -#define MD5C18 0x21e1cde6u -#define MD5C19 0xc33707d6u -#define MD5C1a 0xf4d50d87u -#define MD5C1b 0x455a14edu -#define MD5C1c 0xa9e3e905u -#define MD5C1d 0xfcefa3f8u -#define MD5C1e 0x676f02d9u -#define MD5C1f 0x8d2a4c8au -#define MD5C20 0xfffa3942u -#define MD5C21 0x8771f681u -#define MD5C22 0x6d9d6122u -#define MD5C23 0xfde5380cu -#define MD5C24 0xa4beea44u -#define MD5C25 0x4bdecfa9u -#define MD5C26 0xf6bb4b60u -#define MD5C27 0xbebfbc70u -#define MD5C28 0x289b7ec6u -#define MD5C29 0xeaa127fau -#define MD5C2a 0xd4ef3085u -#define MD5C2b 0x04881d05u -#define MD5C2c 0xd9d4d039u -#define MD5C2d 0xe6db99e5u -#define MD5C2e 0x1fa27cf8u -#define MD5C2f 0xc4ac5665u -#define MD5C30 0xf4292244u -#define MD5C31 0x432aff97u -#define MD5C32 0xab9423a7u -#define MD5C33 0xfc93a039u -#define MD5C34 0x655b59c3u -#define MD5C35 0x8f0ccc92u -#define MD5C36 0xffeff47du -#define MD5C37 0x85845dd1u -#define MD5C38 0x6fa87e4fu -#define MD5C39 0xfe2ce6e0u -#define MD5C3a 0xa3014314u -#define MD5C3b 0x4e0811a1u -#define MD5C3c 0xf7537e82u -#define MD5C3d 0xbd3af235u -#define MD5C3e 0x2ad7d2bbu -#define MD5C3f 0xeb86d391u -#endif - -#if defined _SHA384_ || defined _PDF17L8_ -/** - * SHA384 Constants (64 bits) - */ - -#define SHA384M_A 0xcbbb9d5dc1059ed8 -#define SHA384M_B 0x629a292a367cd507 -#define SHA384M_C 0x9159015a3070dd17 -#define SHA384M_D 0x152fecd8f70e5939 -#define SHA384M_E 0x67332667ffc00b31 -#define SHA384M_F 0x8eb44a8768581511 -#define SHA384M_G 0xdb0c2e0d64f98fa7 -#define SHA384M_H 0x47b5481dbefa4fa4 - -#define SHA384C00 0x428a2f98d728ae22 -#define SHA384C01 0x7137449123ef65cd -#define SHA384C02 0xb5c0fbcfec4d3b2f -#define SHA384C03 0xe9b5dba58189dbbc -#define SHA384C04 0x3956c25bf348b538 -#define SHA384C05 0x59f111f1b605d019 -#define SHA384C06 0x923f82a4af194f9b -#define SHA384C07 0xab1c5ed5da6d8118 -#define SHA384C08 0xd807aa98a3030242 -#define SHA384C09 0x12835b0145706fbe -#define SHA384C0a 0x243185be4ee4b28c -#define SHA384C0b 0x550c7dc3d5ffb4e2 -#define SHA384C0c 0x72be5d74f27b896f -#define SHA384C0d 0x80deb1fe3b1696b1 -#define SHA384C0e 0x9bdc06a725c71235 -#define SHA384C0f 0xc19bf174cf692694 -#define SHA384C10 0xe49b69c19ef14ad2 -#define SHA384C11 0xefbe4786384f25e3 -#define SHA384C12 0x0fc19dc68b8cd5b5 -#define SHA384C13 0x240ca1cc77ac9c65 -#define SHA384C14 0x2de92c6f592b0275 -#define SHA384C15 0x4a7484aa6ea6e483 -#define SHA384C16 0x5cb0a9dcbd41fbd4 -#define SHA384C17 0x76f988da831153b5 -#define SHA384C18 0x983e5152ee66dfab -#define SHA384C19 0xa831c66d2db43210 -#define SHA384C1a 0xb00327c898fb213f -#define SHA384C1b 0xbf597fc7beef0ee4 -#define SHA384C1c 0xc6e00bf33da88fc2 -#define SHA384C1d 0xd5a79147930aa725 -#define SHA384C1e 0x06ca6351e003826f -#define SHA384C1f 0x142929670a0e6e70 -#define SHA384C20 0x27b70a8546d22ffc -#define SHA384C21 0x2e1b21385c26c926 -#define SHA384C22 0x4d2c6dfc5ac42aed -#define SHA384C23 0x53380d139d95b3df -#define SHA384C24 0x650a73548baf63de -#define SHA384C25 0x766a0abb3c77b2a8 -#define SHA384C26 0x81c2c92e47edaee6 -#define SHA384C27 0x92722c851482353b -#define SHA384C28 0xa2bfe8a14cf10364 -#define SHA384C29 0xa81a664bbc423001 -#define SHA384C2a 0xc24b8b70d0f89791 -#define SHA384C2b 0xc76c51a30654be30 -#define SHA384C2c 0xd192e819d6ef5218 -#define SHA384C2d 0xd69906245565a910 -#define SHA384C2e 0xf40e35855771202a -#define SHA384C2f 0x106aa07032bbd1b8 -#define SHA384C30 0x19a4c116b8d2d0c8 -#define SHA384C31 0x1e376c085141ab53 -#define SHA384C32 0x2748774cdf8eeb99 -#define SHA384C33 0x34b0bcb5e19b48a8 -#define SHA384C34 0x391c0cb3c5c95a63 -#define SHA384C35 0x4ed8aa4ae3418acb -#define SHA384C36 0x5b9cca4f7763e373 -#define SHA384C37 0x682e6ff3d6b2b8a3 -#define SHA384C38 0x748f82ee5defb2fc -#define SHA384C39 0x78a5636f43172f60 -#define SHA384C3a 0x84c87814a1f0ab72 -#define SHA384C3b 0x8cc702081a6439ec -#define SHA384C3c 0x90befffa23631e28 -#define SHA384C3d 0xa4506cebde82bde9 -#define SHA384C3e 0xbef9a3f7b2c67915 -#define SHA384C3f 0xc67178f2e372532b -#define SHA384C40 0xca273eceea26619c -#define SHA384C41 0xd186b8c721c0c207 -#define SHA384C42 0xeada7dd6cde0eb1e -#define SHA384C43 0xf57d4f7fee6ed178 -#define SHA384C44 0x06f067aa72176fba -#define SHA384C45 0x0a637dc5a2c898a6 -#define SHA384C46 0x113f9804bef90dae -#define SHA384C47 0x1b710b35131c471b -#define SHA384C48 0x28db77f523047d84 -#define SHA384C49 0x32caab7b40c72493 -#define SHA384C4a 0x3c9ebe0a15c9bebc -#define SHA384C4b 0x431d67c49c100d4c -#define SHA384C4c 0x4cc5d4becb3e42b6 -#define SHA384C4d 0x597f299cfc657e2a -#define SHA384C4e 0x5fcb6fab3ad6faec -#define SHA384C4f 0x6c44198c4a475817 - -#endif - -#if defined _SHA512_ || defined _CLOUDKEY_ || defined _OFFICE2013_ || defined _PDF17L8_ || defined _PBKDF2_SHA512_ -/** - * SHA512 Constants (64 bits) - */ - -#define SHA512M_A 0x6a09e667f3bcc908 -#define SHA512M_B 0xbb67ae8584caa73b -#define SHA512M_C 0x3c6ef372fe94f82b -#define SHA512M_D 0xa54ff53a5f1d36f1 -#define SHA512M_E 0x510e527fade682d1 -#define SHA512M_F 0x9b05688c2b3e6c1f -#define SHA512M_G 0x1f83d9abfb41bd6b -#define SHA512M_H 0x5be0cd19137e2179 - -#define SHA512C00 0x428a2f98d728ae22 -#define SHA512C01 0x7137449123ef65cd -#define SHA512C02 0xb5c0fbcfec4d3b2f -#define SHA512C03 0xe9b5dba58189dbbc -#define SHA512C04 0x3956c25bf348b538 -#define SHA512C05 0x59f111f1b605d019 -#define SHA512C06 0x923f82a4af194f9b -#define SHA512C07 0xab1c5ed5da6d8118 -#define SHA512C08 0xd807aa98a3030242 -#define SHA512C09 0x12835b0145706fbe -#define SHA512C0a 0x243185be4ee4b28c -#define SHA512C0b 0x550c7dc3d5ffb4e2 -#define SHA512C0c 0x72be5d74f27b896f -#define SHA512C0d 0x80deb1fe3b1696b1 -#define SHA512C0e 0x9bdc06a725c71235 -#define SHA512C0f 0xc19bf174cf692694 -#define SHA512C10 0xe49b69c19ef14ad2 -#define SHA512C11 0xefbe4786384f25e3 -#define SHA512C12 0x0fc19dc68b8cd5b5 -#define SHA512C13 0x240ca1cc77ac9c65 -#define SHA512C14 0x2de92c6f592b0275 -#define SHA512C15 0x4a7484aa6ea6e483 -#define SHA512C16 0x5cb0a9dcbd41fbd4 -#define SHA512C17 0x76f988da831153b5 -#define SHA512C18 0x983e5152ee66dfab -#define SHA512C19 0xa831c66d2db43210 -#define SHA512C1a 0xb00327c898fb213f -#define SHA512C1b 0xbf597fc7beef0ee4 -#define SHA512C1c 0xc6e00bf33da88fc2 -#define SHA512C1d 0xd5a79147930aa725 -#define SHA512C1e 0x06ca6351e003826f -#define SHA512C1f 0x142929670a0e6e70 -#define SHA512C20 0x27b70a8546d22ffc -#define SHA512C21 0x2e1b21385c26c926 -#define SHA512C22 0x4d2c6dfc5ac42aed -#define SHA512C23 0x53380d139d95b3df -#define SHA512C24 0x650a73548baf63de -#define SHA512C25 0x766a0abb3c77b2a8 -#define SHA512C26 0x81c2c92e47edaee6 -#define SHA512C27 0x92722c851482353b -#define SHA512C28 0xa2bfe8a14cf10364 -#define SHA512C29 0xa81a664bbc423001 -#define SHA512C2a 0xc24b8b70d0f89791 -#define SHA512C2b 0xc76c51a30654be30 -#define SHA512C2c 0xd192e819d6ef5218 -#define SHA512C2d 0xd69906245565a910 -#define SHA512C2e 0xf40e35855771202a -#define SHA512C2f 0x106aa07032bbd1b8 -#define SHA512C30 0x19a4c116b8d2d0c8 -#define SHA512C31 0x1e376c085141ab53 -#define SHA512C32 0x2748774cdf8eeb99 -#define SHA512C33 0x34b0bcb5e19b48a8 -#define SHA512C34 0x391c0cb3c5c95a63 -#define SHA512C35 0x4ed8aa4ae3418acb -#define SHA512C36 0x5b9cca4f7763e373 -#define SHA512C37 0x682e6ff3d6b2b8a3 -#define SHA512C38 0x748f82ee5defb2fc -#define SHA512C39 0x78a5636f43172f60 -#define SHA512C3a 0x84c87814a1f0ab72 -#define SHA512C3b 0x8cc702081a6439ec -#define SHA512C3c 0x90befffa23631e28 -#define SHA512C3d 0xa4506cebde82bde9 -#define SHA512C3e 0xbef9a3f7b2c67915 -#define SHA512C3f 0xc67178f2e372532b -#define SHA512C40 0xca273eceea26619c -#define SHA512C41 0xd186b8c721c0c207 -#define SHA512C42 0xeada7dd6cde0eb1e -#define SHA512C43 0xf57d4f7fee6ed178 -#define SHA512C44 0x06f067aa72176fba -#define SHA512C45 0x0a637dc5a2c898a6 -#define SHA512C46 0x113f9804bef90dae -#define SHA512C47 0x1b710b35131c471b -#define SHA512C48 0x28db77f523047d84 -#define SHA512C49 0x32caab7b40c72493 -#define SHA512C4a 0x3c9ebe0a15c9bebc -#define SHA512C4b 0x431d67c49c100d4c -#define SHA512C4c 0x4cc5d4becb3e42b6 -#define SHA512C4d 0x597f299cfc657e2a -#define SHA512C4e 0x5fcb6fab3ad6faec -#define SHA512C4f 0x6c44198c4a475817 - -#define SHA512REV0 0x5218a97a1b97e8a0 -#define SHA512REV1 0x4334c1bea164f555 - -#endif - -#ifdef _RIPEMD160_ -/** - * RIPEMD160 Constants - */ - -#define RIPEMD160M_A 0x67452301u -#define RIPEMD160M_B 0xefcdab89u -#define RIPEMD160M_C 0x98badcfeu -#define RIPEMD160M_D 0x10325476u -#define RIPEMD160M_E 0xc3d2e1f0u - -#define RIPEMD160C00 0x00000000u -#define RIPEMD160C10 0x5a827999u -#define RIPEMD160C20 0x6ed9eba1u -#define RIPEMD160C30 0x8f1bbcdcu -#define RIPEMD160C40 0xa953fd4eu -#define RIPEMD160C50 0x50a28be6u -#define RIPEMD160C60 0x5c4dd124u -#define RIPEMD160C70 0x6d703ef3u -#define RIPEMD160C80 0x7a6d76e9u -#define RIPEMD160C90 0x00000000u - -#define RIPEMD160S00 11u -#define RIPEMD160S01 14u -#define RIPEMD160S02 15u -#define RIPEMD160S03 12u -#define RIPEMD160S04 5u -#define RIPEMD160S05 8u -#define RIPEMD160S06 7u -#define RIPEMD160S07 9u -#define RIPEMD160S08 11u -#define RIPEMD160S09 13u -#define RIPEMD160S0A 14u -#define RIPEMD160S0B 15u -#define RIPEMD160S0C 6u -#define RIPEMD160S0D 7u -#define RIPEMD160S0E 9u -#define RIPEMD160S0F 8u - -#define RIPEMD160S10 7u -#define RIPEMD160S11 6u -#define RIPEMD160S12 8u -#define RIPEMD160S13 13u -#define RIPEMD160S14 11u -#define RIPEMD160S15 9u -#define RIPEMD160S16 7u -#define RIPEMD160S17 15u -#define RIPEMD160S18 7u -#define RIPEMD160S19 12u -#define RIPEMD160S1A 15u -#define RIPEMD160S1B 9u -#define RIPEMD160S1C 11u -#define RIPEMD160S1D 7u -#define RIPEMD160S1E 13u -#define RIPEMD160S1F 12u - -#define RIPEMD160S20 11u -#define RIPEMD160S21 13u -#define RIPEMD160S22 6u -#define RIPEMD160S23 7u -#define RIPEMD160S24 14u -#define RIPEMD160S25 9u -#define RIPEMD160S26 13u -#define RIPEMD160S27 15u -#define RIPEMD160S28 14u -#define RIPEMD160S29 8u -#define RIPEMD160S2A 13u -#define RIPEMD160S2B 6u -#define RIPEMD160S2C 5u -#define RIPEMD160S2D 12u -#define RIPEMD160S2E 7u -#define RIPEMD160S2F 5u - -#define RIPEMD160S30 11u -#define RIPEMD160S31 12u -#define RIPEMD160S32 14u -#define RIPEMD160S33 15u -#define RIPEMD160S34 14u -#define RIPEMD160S35 15u -#define RIPEMD160S36 9u -#define RIPEMD160S37 8u -#define RIPEMD160S38 9u -#define RIPEMD160S39 14u -#define RIPEMD160S3A 5u -#define RIPEMD160S3B 6u -#define RIPEMD160S3C 8u -#define RIPEMD160S3D 6u -#define RIPEMD160S3E 5u -#define RIPEMD160S3F 12u - -#define RIPEMD160S40 9u -#define RIPEMD160S41 15u -#define RIPEMD160S42 5u -#define RIPEMD160S43 11u -#define RIPEMD160S44 6u -#define RIPEMD160S45 8u -#define RIPEMD160S46 13u -#define RIPEMD160S47 12u -#define RIPEMD160S48 5u -#define RIPEMD160S49 12u -#define RIPEMD160S4A 13u -#define RIPEMD160S4B 14u -#define RIPEMD160S4C 11u -#define RIPEMD160S4D 8u -#define RIPEMD160S4E 5u -#define RIPEMD160S4F 6u - -#define RIPEMD160S50 8u -#define RIPEMD160S51 9u -#define RIPEMD160S52 9u -#define RIPEMD160S53 11u -#define RIPEMD160S54 13u -#define RIPEMD160S55 15u -#define RIPEMD160S56 15u -#define RIPEMD160S57 5u -#define RIPEMD160S58 7u -#define RIPEMD160S59 7u -#define RIPEMD160S5A 8u -#define RIPEMD160S5B 11u -#define RIPEMD160S5C 14u -#define RIPEMD160S5D 14u -#define RIPEMD160S5E 12u -#define RIPEMD160S5F 6u - -#define RIPEMD160S60 9u -#define RIPEMD160S61 13u -#define RIPEMD160S62 15u -#define RIPEMD160S63 7u -#define RIPEMD160S64 12u -#define RIPEMD160S65 8u -#define RIPEMD160S66 9u -#define RIPEMD160S67 11u -#define RIPEMD160S68 7u -#define RIPEMD160S69 7u -#define RIPEMD160S6A 12u -#define RIPEMD160S6B 7u -#define RIPEMD160S6C 6u -#define RIPEMD160S6D 15u -#define RIPEMD160S6E 13u -#define RIPEMD160S6F 11u - -#define RIPEMD160S70 9u -#define RIPEMD160S71 7u -#define RIPEMD160S72 15u -#define RIPEMD160S73 11u -#define RIPEMD160S74 8u -#define RIPEMD160S75 6u -#define RIPEMD160S76 6u -#define RIPEMD160S77 14u -#define RIPEMD160S78 12u -#define RIPEMD160S79 13u -#define RIPEMD160S7A 5u -#define RIPEMD160S7B 14u -#define RIPEMD160S7C 13u -#define RIPEMD160S7D 13u -#define RIPEMD160S7E 7u -#define RIPEMD160S7F 5u - -#define RIPEMD160S80 15u -#define RIPEMD160S81 5u -#define RIPEMD160S82 8u -#define RIPEMD160S83 11u -#define RIPEMD160S84 14u -#define RIPEMD160S85 14u -#define RIPEMD160S86 6u -#define RIPEMD160S87 14u -#define RIPEMD160S88 6u -#define RIPEMD160S89 9u -#define RIPEMD160S8A 12u -#define RIPEMD160S8B 9u -#define RIPEMD160S8C 12u -#define RIPEMD160S8D 5u -#define RIPEMD160S8E 15u -#define RIPEMD160S8F 8u - -#define RIPEMD160S90 8u -#define RIPEMD160S91 5u -#define RIPEMD160S92 12u -#define RIPEMD160S93 9u -#define RIPEMD160S94 12u -#define RIPEMD160S95 5u -#define RIPEMD160S96 14u -#define RIPEMD160S97 6u -#define RIPEMD160S98 8u -#define RIPEMD160S99 13u -#define RIPEMD160S9A 6u -#define RIPEMD160S9B 5u -#define RIPEMD160S9C 15u -#define RIPEMD160S9D 13u -#define RIPEMD160S9E 11u -#define RIPEMD160S9F 11u - -#endif - -#ifdef _KECCAK_ -/** - * KECCAK Constants - */ - -#define KECCAK_RNDC_00 0x0000000000000001 -#define KECCAK_RNDC_01 0x0000000000008082 -#define KECCAK_RNDC_02 0x000000000000808a -#define KECCAK_RNDC_03 0x0000000080008000 -#define KECCAK_RNDC_04 0x000000000000808b -#define KECCAK_RNDC_05 0x0000000080000001 -#define KECCAK_RNDC_06 0x0000000080008081 -#define KECCAK_RNDC_07 0x0000000000008009 -#define KECCAK_RNDC_08 0x000000000000008a -#define KECCAK_RNDC_09 0x0000000000000088 -#define KECCAK_RNDC_10 0x0000000080008009 -#define KECCAK_RNDC_11 0x000000008000000a -#define KECCAK_RNDC_12 0x000000008000808b -#define KECCAK_RNDC_13 0x000000000000008b -#define KECCAK_RNDC_14 0x0000000000008089 -#define KECCAK_RNDC_15 0x0000000000008003 -#define KECCAK_RNDC_16 0x0000000000008002 -#define KECCAK_RNDC_17 0x0000000000000080 -#define KECCAK_RNDC_18 0x000000000000800a -#define KECCAK_RNDC_19 0x000000008000000a -#define KECCAK_RNDC_20 0x0000000080008081 -#define KECCAK_RNDC_21 0x0000000000008080 -#define KECCAK_RNDC_22 0x0000000080000001 -#define KECCAK_RNDC_23 0x0000000080008008 - -#define KECCAK_PILN_00 10 -#define KECCAK_PILN_01 7 -#define KECCAK_PILN_02 11 -#define KECCAK_PILN_03 17 -#define KECCAK_PILN_04 18 -#define KECCAK_PILN_05 3 -#define KECCAK_PILN_06 5 -#define KECCAK_PILN_07 16 -#define KECCAK_PILN_08 8 -#define KECCAK_PILN_09 21 -#define KECCAK_PILN_10 24 -#define KECCAK_PILN_11 4 -#define KECCAK_PILN_12 15 -#define KECCAK_PILN_13 23 -#define KECCAK_PILN_14 19 -#define KECCAK_PILN_15 13 -#define KECCAK_PILN_16 12 -#define KECCAK_PILN_17 2 -#define KECCAK_PILN_18 20 -#define KECCAK_PILN_19 14 -#define KECCAK_PILN_20 22 -#define KECCAK_PILN_21 9 -#define KECCAK_PILN_22 6 -#define KECCAK_PILN_23 1 - -#define KECCAK_ROTC_00 1 -#define KECCAK_ROTC_01 3 -#define KECCAK_ROTC_02 6 -#define KECCAK_ROTC_03 10 -#define KECCAK_ROTC_04 15 -#define KECCAK_ROTC_05 21 -#define KECCAK_ROTC_06 28 -#define KECCAK_ROTC_07 36 -#define KECCAK_ROTC_08 45 -#define KECCAK_ROTC_09 55 -#define KECCAK_ROTC_10 2 -#define KECCAK_ROTC_11 14 -#define KECCAK_ROTC_12 27 -#define KECCAK_ROTC_13 41 -#define KECCAK_ROTC_14 56 -#define KECCAK_ROTC_15 8 -#define KECCAK_ROTC_16 25 -#define KECCAK_ROTC_17 43 -#define KECCAK_ROTC_18 62 -#define KECCAK_ROTC_19 18 -#define KECCAK_ROTC_20 39 -#define KECCAK_ROTC_21 61 -#define KECCAK_ROTC_22 20 -#define KECCAK_ROTC_23 44 - - -#endif - -#ifdef _MYSQL323_ -/** - * MYSQL323 Constants - */ - -#define MYSQL323_A 0x50305735 -#define MYSQL323_B 0x12345671 - -#endif diff --git a/include/kernel_functions.c b/include/kernel_functions.c deleted file mode 100644 index 0ccbb25..0000000 --- a/include/kernel_functions.c +++ /dev/null @@ -1,413 +0,0 @@ -/** - * Author......: Jens Steube - * License.....: MIT - */ - -#if defined _MD4_ || defined _DCC2_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _MS_DRSR_ || defined _KRB5TGS_ - -#define MD4_F_S(x,y,z) (((x) & (y)) | ((~(x)) & (z))) -#define MD4_G_S(x,y,z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) -#define MD4_H_S(x,y,z) ((x) ^ (y) ^ (z)) - -#ifdef IS_NV -#define MD4_F(x,y,z) (((x) & (y)) | ((~(x)) & (z))) -#define MD4_G(x,y,z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) -#define MD4_H(x,y,z) ((x) ^ (y) ^ (z)) -#define MD4_Fo(x,y,z) (MD4_F((x), (y), (z))) -#define MD4_Go(x,y,z) (MD4_G((x), (y), (z))) -#endif - -#ifdef IS_AMD -#define MD4_F(x,y,z) (((x) & (y)) | ((~(x)) & (z))) -#define MD4_G(x,y,z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) -#define MD4_H(x,y,z) ((x) ^ (y) ^ (z)) -#define MD4_Fo(x,y,z) (bitselect ((z), (y), (x))) -#define MD4_Go(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) -#endif - -#ifdef IS_GENERIC -#define MD4_F(x,y,z) (((x) & (y)) | ((~(x)) & (z))) -#define MD4_G(x,y,z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) -#define MD4_H(x,y,z) ((x) ^ (y) ^ (z)) -#define MD4_Fo(x,y,z) (MD4_F((x), (y), (z))) -#define MD4_Go(x,y,z) (MD4_G((x), (y), (z))) -#endif - -#define MD4_STEP_S(f,a,b,c,d,x,K,s) \ -{ \ - a += K; \ - a += x; \ - a += f (b, c, d); \ - a = rotl32_S (a, s); \ -} - -#define MD4_STEP(f,a,b,c,d,x,K,s) \ -{ \ - a += K; \ - a += x; \ - a += f (b, c, d); \ - a = rotl32 (a, s); \ -} - -#define MD4_STEP0(f,a,b,c,d,K,s) \ -{ \ - a += K; \ - a += f (b, c, d); \ - a = rotl32 (a, s); \ -} - -#endif - -#if defined _MD5_ || defined _MD5H_ || defined _SAPB_ || defined _OLDOFFICE01_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _PBKDF2_MD5_ || defined _KRB5TGS_ - -#define MD5_F_S(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define MD5_G_S(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) -#define MD5_H_S(x,y,z) ((x) ^ (y) ^ (z)) -#define MD5_I_S(x,y,z) ((y) ^ ((x) | ~(z))) - -#ifdef IS_NV -#define MD5_F(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define MD5_G(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) -#define MD5_H(x,y,z) ((x) ^ (y) ^ (z)) -#define MD5_I(x,y,z) ((y) ^ ((x) | ~(z))) -#define MD5_Fo(x,y,z) (MD5_F((x), (y), (z))) -#define MD5_Go(x,y,z) (MD5_G((x), (y), (z))) -#endif - -#ifdef IS_AMD -#define MD5_F(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define MD5_G(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) -#define MD5_H(x,y,z) ((x) ^ (y) ^ (z)) -#define MD5_I(x,y,z) (bitselect (0xffffffffU, (x), (z)) ^ (y)) -#define MD5_Fo(x,y,z) (bitselect ((z), (y), (x))) -#define MD5_Go(x,y,z) (bitselect ((y), (x), (z))) -#endif - -#ifdef IS_GENERIC -#define MD5_F(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define MD5_G(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) -#define MD5_H(x,y,z) ((x) ^ (y) ^ (z)) -#define MD5_I(x,y,z) ((y) ^ ((x) | ~(z))) -#define MD5_Fo(x,y,z) (MD5_F((x), (y), (z))) -#define MD5_Go(x,y,z) (MD5_G((x), (y), (z))) -#endif - -#define MD5_STEP_S(f,a,b,c,d,x,K,s) \ -{ \ - a += K; \ - a += x; \ - a += f (b, c, d); \ - a = rotl32_S (a, s); \ - a += b; \ -} - -#define MD5_STEP(f,a,b,c,d,x,K,s) \ -{ \ - a += K; \ - a += x; \ - a += f (b, c, d); \ - a = rotl32 (a, s); \ - a += b; \ -} - -#define MD5_STEP0(f,a,b,c,d,K,s) \ -{ \ - a += K; \ - a += f (b, c, d); \ - a = rotl32 (a, s); \ - a += b; \ -} -#endif - -#if defined _SHA1_ || defined _SAPG_ || defined _OFFICE2007_ || defined _OFFICE2010_ || defined _OLDOFFICE34_ || defined _ANDROIDFDE_ || defined _DCC2_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _PSAFE2_ || defined _LOTUS8_ || defined _PBKDF2_SHA1_ || defined _RAR3_ || defined _SHA256_SHA1_ || defined _ZIP2_ - -#ifdef IS_NV -#define SHA1_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define SHA1_F1(x,y,z) ((x) ^ (y) ^ (z)) -#define SHA1_F2(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) -#define SHA1_F0o(x,y,z) (SHA1_F0 ((x), (y), (z))) -#define SHA1_F2o(x,y,z) (SHA1_F2 ((x), (y), (z))) -#endif - -#ifdef IS_AMD -#define SHA1_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define SHA1_F1(x,y,z) ((x) ^ (y) ^ (z)) -#define SHA1_F2(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) -#define SHA1_F0o(x,y,z) (bitselect ((z), (y), (x))) -#define SHA1_F2o(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) -#endif - -#ifdef IS_GENERIC -#define SHA1_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define SHA1_F1(x,y,z) ((x) ^ (y) ^ (z)) -#define SHA1_F2(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) -#define SHA1_F0o(x,y,z) (SHA1_F0 ((x), (y), (z))) -#define SHA1_F2o(x,y,z) (SHA1_F2 ((x), (y), (z))) -#endif - -#define SHA1_STEP_S(f,a,b,c,d,e,x) \ -{ \ - e += K; \ - e += x; \ - e += f (b, c, d); \ - e += rotl32_S (a, 5u); \ - b = rotl32_S (b, 30u); \ -} - -#define SHA1_STEP(f,a,b,c,d,e,x) \ -{ \ - e += K; \ - e += x; \ - e += f (b, c, d); \ - e += rotl32 (a, 5u); \ - b = rotl32 (b, 30u); \ -} - -#define SHA1_STEP0(f,a,b,c,d,e,x) \ -{ \ - e += K; \ - e += f (b, c, d); \ - e += rotl32 (a, 5u); \ - b = rotl32 (b, 30u); \ -} - -#define SHA1_STEPX(f,a,b,c,d,e,x) \ -{ \ - e += x; \ - e += f (b, c, d); \ - e += rotl32 (a, 5u); \ - b = rotl32 (b, 30u); \ -} - -#define SHA1_STEP_PE(f,a,b,c,d,e,x) \ -{ \ - e += x; \ - e += f (b, c, d); \ - e += rotl32 (a, 5u); \ -} - -#define SHA1_STEP_PB(f,a,b,c,d,e,x) \ -{ \ - e += K; \ - b = rotl32 (b, 30u); \ -} -#endif - -#if defined _SHA256_ || defined _PDF17L8_ || defined _SEVEN_ZIP_ || defined _ANDROIDFDE_ || defined _CLOUDKEY_ || defined _SCRYPT_ || defined _PBKDF2_SHA256_ || defined _SHA256_SHA1_ || defined _MS_DRSR_ || defined _ANDROIDFDE_SAMSUNG_ || defined _RAR5_ || defined _KEEPASS_ - -#define SHIFT_RIGHT_32(x,n) ((x) >> (n)) - -#define SHA256_S0_S(x) (rotl32_S ((x), 25u) ^ rotl32_S ((x), 14u) ^ SHIFT_RIGHT_32 ((x), 3u)) -#define SHA256_S1_S(x) (rotl32_S ((x), 15u) ^ rotl32_S ((x), 13u) ^ SHIFT_RIGHT_32 ((x), 10u)) -#define SHA256_S2_S(x) (rotl32_S ((x), 30u) ^ rotl32_S ((x), 19u) ^ rotl32_S ((x), 10u)) -#define SHA256_S3_S(x) (rotl32_S ((x), 26u) ^ rotl32_S ((x), 21u) ^ rotl32_S ((x), 7u)) - -#define SHA256_S0(x) (rotl32 ((x), 25u) ^ rotl32 ((x), 14u) ^ SHIFT_RIGHT_32 ((x), 3u)) -#define SHA256_S1(x) (rotl32 ((x), 15u) ^ rotl32 ((x), 13u) ^ SHIFT_RIGHT_32 ((x), 10u)) -#define SHA256_S2(x) (rotl32 ((x), 30u) ^ rotl32 ((x), 19u) ^ rotl32 ((x), 10u)) -#define SHA256_S3(x) (rotl32 ((x), 26u) ^ rotl32 ((x), 21u) ^ rotl32 ((x), 7u)) - -#ifdef IS_NV -#define SHA256_F0(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) -#define SHA256_F1(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define SHA256_F0o(x,y,z) (SHA256_F0 ((x), (y), (z))) -#define SHA256_F1o(x,y,z) (SHA256_F1 ((x), (y), (z))) -#endif - -#ifdef IS_AMD -#define SHA256_F0(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) -#define SHA256_F1(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define SHA256_F0o(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) -#define SHA256_F1o(x,y,z) (bitselect ((z), (y), (x))) -#endif - -#ifdef IS_GENERIC -#define SHA256_F0(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) -#define SHA256_F1(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define SHA256_F0o(x,y,z) (SHA256_F0 ((x), (y), (z))) -#define SHA256_F1o(x,y,z) (SHA256_F1 ((x), (y), (z))) -#endif - -#define SHA256_STEP_S(F0,F1,a,b,c,d,e,f,g,h,x,K) \ -{ \ - h += K; \ - h += x; \ - h += SHA256_S3_S (e); \ - h += F1 (e,f,g); \ - d += h; \ - h += SHA256_S2_S (a); \ - h += F0 (a,b,c); \ -} - -#define SHA256_EXPAND_S(x,y,z,w) (SHA256_S1_S (x) + y + SHA256_S0_S (z) + w) - -#define SHA256_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K) \ -{ \ - h += K; \ - h += x; \ - h += SHA256_S3 (e); \ - h += F1 (e,f,g); \ - d += h; \ - h += SHA256_S2 (a); \ - h += F0 (a,b,c); \ -} - -#define SHA256_EXPAND(x,y,z,w) (SHA256_S1 (x) + y + SHA256_S0 (z) + w) - -#endif - -#if defined _SHA384_ || defined _PDF17L8_ - -#define SHIFT_RIGHT_64(x,n) ((x) >> (n)) - -#define SHA384_S0(x) (rotr64 ((x), 28) ^ rotr64 ((x), 34) ^ rotr64 ((x), 39)) -#define SHA384_S1(x) (rotr64 ((x), 14) ^ rotr64 ((x), 18) ^ rotr64 ((x), 41)) -#define SHA384_S2(x) (rotr64 ((x), 1) ^ rotr64 ((x), 8) ^ SHIFT_RIGHT_64 ((x), 7)) -#define SHA384_S3(x) (rotr64 ((x), 19) ^ rotr64 ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6)) - -#define SHA384_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define SHA384_F1(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) - -#ifdef IS_NV -#define SHA384_F0o(x,y,z) (SHA384_F0 ((x), (y), (z))) -#define SHA384_F1o(x,y,z) (SHA384_F1 ((x), (y), (z))) -#endif - -#ifdef IS_AMD -#define SHA384_F0o(x,y,z) (bitselect ((z), (y), (x))) -#define SHA384_F1o(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) -#endif - -#ifdef IS_GENERIC -#define SHA384_F0o(x,y,z) (SHA384_F0 ((x), (y), (z))) -#define SHA384_F1o(x,y,z) (SHA384_F1 ((x), (y), (z))) -#endif - -#define SHA384_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K) \ -{ \ - h += K; \ - h += x; \ - h += SHA384_S1 (e); \ - h += F0 (e, f, g); \ - d += h; \ - h += SHA384_S0 (a); \ - h += F1 (a, b, c); \ -} - -#define SHA384_EXPAND(x,y,z,w) (SHA384_S3 (x) + y + SHA384_S2 (z) + w) -#endif - -#if defined _SHA512_ || defined _CLOUDKEY_ || defined _OFFICE2013_ || defined _PDF17L8_ || defined _PBKDF2_SHA512_ - -#define SHIFT_RIGHT_64(x,n) ((x) >> (n)) - -#define SHA512_S0_S(x) (rotr64_S ((x), 28) ^ rotr64_S ((x), 34) ^ rotr64_S ((x), 39)) -#define SHA512_S1_S(x) (rotr64_S ((x), 14) ^ rotr64_S ((x), 18) ^ rotr64_S ((x), 41)) -#define SHA512_S2_S(x) (rotr64_S ((x), 1) ^ rotr64_S ((x), 8) ^ SHIFT_RIGHT_64 ((x), 7)) -#define SHA512_S3_S(x) (rotr64_S ((x), 19) ^ rotr64_S ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6)) - -#define SHA512_S0(x) (rotr64 ((x), 28) ^ rotr64 ((x), 34) ^ rotr64 ((x), 39)) -#define SHA512_S1(x) (rotr64 ((x), 14) ^ rotr64 ((x), 18) ^ rotr64 ((x), 41)) -#define SHA512_S2(x) (rotr64 ((x), 1) ^ rotr64 ((x), 8) ^ SHIFT_RIGHT_64 ((x), 7)) -#define SHA512_S3(x) (rotr64 ((x), 19) ^ rotr64 ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6)) - -#define SHA512_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define SHA512_F1(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y)))) - -#ifdef IS_NV -#define SHA512_F0o(x,y,z) (SHA512_F0 ((x), (y), (z))) -#define SHA512_F1o(x,y,z) (SHA512_F1 ((x), (y), (z))) -#endif - -#ifdef IS_AMD -#define SHA512_F0o(x,y,z) (bitselect ((z), (y), (x))) -#define SHA512_F1o(x,y,z) (bitselect ((x), (y), ((x) ^ (z)))) -#endif - -#ifdef IS_GENERIC -#define SHA512_F0o(x,y,z) (SHA512_F0 ((x), (y), (z))) -#define SHA512_F1o(x,y,z) (SHA512_F1 ((x), (y), (z))) -#endif - -#define SHA512_STEP_S(F0,F1,a,b,c,d,e,f,g,h,x,K) \ -{ \ - h += K; \ - h += x; \ - h += SHA512_S1_S (e); \ - h += F0 (e, f, g); \ - d += h; \ - h += SHA512_S0_S (a); \ - h += F1 (a, b, c); \ -} - -#define SHA512_EXPAND_S(x,y,z,w) (SHA512_S3_S (x) + y + SHA512_S2_S (z) + w) - -#define SHA512_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K) \ -{ \ - h += K; \ - h += x; \ - h += SHA512_S1 (e); \ - h += F0 (e, f, g); \ - d += h; \ - h += SHA512_S0 (a); \ - h += F1 (a, b, c); \ -} - -#define SHA512_EXPAND(x,y,z,w) (SHA512_S3 (x) + y + SHA512_S2 (z) + w) -#endif - -#ifdef _RIPEMD160_ - -#ifdef IS_NV -#define RIPEMD160_F(x,y,z) ((x) ^ (y) ^ (z)) -#define RIPEMD160_G(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */ -#define RIPEMD160_H(x,y,z) (((x) | ~(y)) ^ (z)) -#define RIPEMD160_I(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */ -#define RIPEMD160_J(x,y,z) ((x) ^ ((y) | ~(z))) -#define RIPEMD160_Go(x,y,z) (RIPEMD160_G ((x), (y), (z))) -#define RIPEMD160_Io(x,y,z) (RIPEMD160_I ((x), (y), (z))) -#endif - -#ifdef IS_AMD -#define RIPEMD160_F(x,y,z) ((x) ^ (y) ^ (z)) -#define RIPEMD160_G(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */ -#define RIPEMD160_H(x,y,z) (((x) | ~(y)) ^ (z)) -#define RIPEMD160_I(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */ -#define RIPEMD160_J(x,y,z) ((x) ^ ((y) | ~(z))) -#define RIPEMD160_Go(x,y,z) (bitselect ((z), (y), (x))) -#define RIPEMD160_Io(x,y,z) (bitselect ((y), (x), (z))) -#endif - -#ifdef IS_GENERIC -#define RIPEMD160_F(x,y,z) ((x) ^ (y) ^ (z)) -#define RIPEMD160_G(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */ -#define RIPEMD160_H(x,y,z) (((x) | ~(y)) ^ (z)) -#define RIPEMD160_I(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */ -#define RIPEMD160_J(x,y,z) ((x) ^ ((y) | ~(z))) -#define RIPEMD160_Go(x,y,z) (RIPEMD160_G ((x), (y), (z))) -#define RIPEMD160_Io(x,y,z) (RIPEMD160_I ((x), (y), (z))) -#endif - -#define RIPEMD160_STEP(f,a,b,c,d,e,x,K,s) \ -{ \ - a += K; \ - a += x; \ - a += f (b, c, d); \ - a = rotl32 (a, s); \ - a += e; \ - c = rotl32 (c, 10u); \ -} - -#define ROTATE_LEFT_WORKAROUND_BUG(a,n) ((a << n) | (a >> (32 - n))) - -#define RIPEMD160_STEP_WORKAROUND_BUG(f,a,b,c,d,e,x,K,s) \ -{ \ - a += K; \ - a += x; \ - a += f (b, c, d); \ - a = ROTATE_LEFT_WORKAROUND_BUG (a, s); \ - a += e; \ - c = rotl32 (c, 10u); \ -} - -#endif diff --git a/include/kernel_vendor.h b/include/kernel_vendor.h deleted file mode 100644 index 3adde96..0000000 --- a/include/kernel_vendor.h +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Author......: Jens Steube - * License.....: MIT - */ - -//fails on intel opencl sdk -//#pragma OPENCL EXTENSION cl_khr_int64_base_atomics : enable -//#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable - -/** - * device type - */ - -#define DEVICE_TYPE_CPU 2 -#define DEVICE_TYPE_GPU 4 -#define DEVICE_TYPE_ACCEL 8 - -#if DEVICE_TYPE == DEVICE_TYPE_CPU -#define IS_CPU -#elif DEVICE_TYPE == DEVICE_TYPE_GPU -#define IS_GPU -#elif DEVICE_TYPE == DEVICE_TYPE_ACCEL -#define IS_ACCEL -#endif - -/** - * vendor specific - */ - -#if VENDOR_ID == (1 << 0) -#define IS_AMD -//#define IS_GENERIC -#elif VENDOR_ID == (1 << 1) -#define IS_APPLE -#define IS_GENERIC -#elif VENDOR_ID == (1 << 2) -#define IS_INTEL_BEIGNET -#define IS_GENERIC -#elif VENDOR_ID == (1 << 3) -#define IS_INTEL_SDK -#define IS_GENERIC -#elif VENDOR_ID == (1 << 4) -#define IS_MESA -#define IS_GENERIC -#elif VENDOR_ID == (1 << 5) -#define IS_NV -//#define IS_GENERIC -#elif VENDOR_ID == (1 << 6) -#define IS_POCL -#define IS_GENERIC -#else -#define IS_GENERIC -#endif - -/** - * AMD specific - */ - -#ifdef IS_AMD -#pragma OPENCL EXTENSION cl_amd_media_ops : enable -#pragma OPENCL EXTENSION cl_amd_media_ops2 : enable -#endif - -/** - * Unrolling is generally enabled, for all device types and hash modes - * There's a few exception when it's better not to unroll - */ - -// Some algorithms run into too much register pressure due to loop unrolling - -#ifdef IS_NV -#ifdef IS_GPU - -#if KERN_TYPE == 1500 -#undef _unroll -#endif -#if KERN_TYPE == 1800 -#undef _unroll -#endif -#if KERN_TYPE == 3000 -#undef _unroll -#endif -#if KERN_TYPE == 6221 -#undef _unroll -#endif -#if KERN_TYPE == 6222 -#undef _unroll -#endif -#if KERN_TYPE == 6223 -#undef _unroll -#endif -#if KERN_TYPE == 6500 -#undef _unroll -#endif -#if KERN_TYPE == 7100 -#undef _unroll -#endif -#if KERN_TYPE == 7400 -#undef _unroll -#endif -#if KERN_TYPE == 7900 -#undef _unroll -#endif -#if KERN_TYPE == 8200 -#undef _unroll -#endif -#if KERN_TYPE == 10400 -#undef _unroll -#endif -#if KERN_TYPE == 10500 -#undef _unroll -#endif -#if KERN_TYPE == 10700 -#undef _unroll -#endif -#if KERN_TYPE == 12300 -#undef _unroll -#endif -#if KERN_TYPE == 12400 -#undef _unroll -#endif -#if KERN_TYPE == 13721 -#undef _unroll -#endif -#if KERN_TYPE == 13722 -#undef _unroll -#endif -#if KERN_TYPE == 13723 -#undef _unroll -#endif - -#endif -#endif - -#ifdef IS_AMD -#ifdef IS_GPU - -#if KERN_TYPE == 3200 -#undef _unroll -#endif -#if KERN_TYPE == 5200 -#undef _unroll -#endif -#if KERN_TYPE == 6100 -#undef _unroll -#endif -#if KERN_TYPE == 6221 -#undef _unroll -#endif -#if KERN_TYPE == 6222 -#undef _unroll -#endif -#if KERN_TYPE == 6223 -#undef _unroll -#endif -#if KERN_TYPE == 6400 -#undef _unroll -#endif -#if KERN_TYPE == 6500 -#undef _unroll -#endif -#if KERN_TYPE == 6800 -#undef _unroll -#endif -#if KERN_TYPE == 7100 -#undef _unroll -#endif -#if KERN_TYPE == 7400 -#undef _unroll -#endif -#if KERN_TYPE == 8000 -#undef _unroll -#endif -#if KERN_TYPE == 8200 -#undef _unroll -#endif -#if KERN_TYPE == 10900 -#undef _unroll -#endif -#if KERN_TYPE == 11600 -#undef _unroll -#endif -#if KERN_TYPE == 12300 -#undef _unroll -#endif -#if KERN_TYPE == 12800 -#undef _unroll -#endif -#if KERN_TYPE == 12900 -#undef _unroll -#endif -#if KERN_TYPE == 13000 -#undef _unroll -#endif -#if KERN_TYPE == 13721 -#undef _unroll -#endif -#if KERN_TYPE == 13722 -#undef _unroll -#endif -#if KERN_TYPE == 13723 -#undef _unroll -#endif - -#endif -#endif - -// Some algorithms break due to loop unrolling, it's unknown why, probably compiler bugs -// Can overlap with above cases - -#ifdef IS_AMD -#ifdef IS_GPU - -#if KERN_TYPE == 1750 -#undef _unroll -#endif -#if KERN_TYPE == 1760 -#undef _unroll -#endif -#if KERN_TYPE == 6500 -#undef _unroll -#endif -#if KERN_TYPE == 7100 -#undef _unroll -#endif -#if KERN_TYPE == 9600 -#undef _unroll -#endif -#if KERN_TYPE == 12200 -#undef _unroll -#endif -#if KERN_TYPE == 12300 -#undef _unroll -#endif - -#endif -#endif diff --git a/include/rp_kernel.h b/include/rp_kernel.h deleted file mode 100644 index 17849f2..0000000 --- a/include/rp_kernel.h +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Author......: Jens Steube - * License.....: MIT - */ - -#define RULE_OP_MANGLE_NOOP ':' -#define RULE_OP_MANGLE_LREST 'l' -#define RULE_OP_MANGLE_UREST 'u' -#define RULE_OP_MANGLE_LREST_UFIRST 'c' -#define RULE_OP_MANGLE_UREST_LFIRST 'C' -#define RULE_OP_MANGLE_TREST 't' -#define RULE_OP_MANGLE_TOGGLE_AT 'T' -#define RULE_OP_MANGLE_REVERSE 'r' -#define RULE_OP_MANGLE_DUPEWORD 'd' -#define RULE_OP_MANGLE_DUPEWORD_TIMES 'p' -#define RULE_OP_MANGLE_REFLECT 'f' -#define RULE_OP_MANGLE_ROTATE_LEFT '{' -#define RULE_OP_MANGLE_ROTATE_RIGHT '}' -#define RULE_OP_MANGLE_APPEND '$' -#define RULE_OP_MANGLE_PREPEND '^' -#define RULE_OP_MANGLE_DELETE_FIRST '[' -#define RULE_OP_MANGLE_DELETE_LAST ']' -#define RULE_OP_MANGLE_DELETE_AT 'D' -#define RULE_OP_MANGLE_EXTRACT 'x' -#define RULE_OP_MANGLE_OMIT 'O' -#define RULE_OP_MANGLE_INSERT 'i' -#define RULE_OP_MANGLE_OVERSTRIKE 'o' -#define RULE_OP_MANGLE_TRUNCATE_AT '\'' -#define RULE_OP_MANGLE_REPLACE 's' -#define RULE_OP_MANGLE_PURGECHAR '@' -#define RULE_OP_MANGLE_TOGGLECASE_REC 'a' -#define RULE_OP_MANGLE_DUPECHAR_FIRST 'z' -#define RULE_OP_MANGLE_DUPECHAR_LAST 'Z' -#define RULE_OP_MANGLE_DUPECHAR_ALL 'q' - -#define RULE_OP_REJECT_LESS '<' -#define RULE_OP_REJECT_GREATER '>' -#define RULE_OP_REJECT_CONTAIN '!' -#define RULE_OP_REJECT_NOT_CONTAIN '/' -#define RULE_OP_REJECT_EQUAL_FIRST '(' -#define RULE_OP_REJECT_EQUAL_LAST ')' -#define RULE_OP_REJECT_EQUAL_AT '=' -#define RULE_OP_REJECT_CONTAINS '%' - -/* hashcat only */ -#define RULE_OP_MANGLE_SWITCH_FIRST 'k' -#define RULE_OP_MANGLE_SWITCH_LAST 'K' -#define RULE_OP_MANGLE_SWITCH_AT '*' -#define RULE_OP_MANGLE_CHR_SHIFTL 'L' -#define RULE_OP_MANGLE_CHR_SHIFTR 'R' -#define RULE_OP_MANGLE_CHR_INCR '+' -#define RULE_OP_MANGLE_CHR_DECR '-' -#define RULE_OP_MANGLE_REPLACE_NP1 '.' -#define RULE_OP_MANGLE_REPLACE_NM1 ',' -#define RULE_OP_MANGLE_DUPEBLOCK_FIRST 'y' -#define RULE_OP_MANGLE_DUPEBLOCK_LAST 'Y' -#define RULE_OP_MANGLE_TITLE 'E' diff --git a/include/rp_kernel_on_cpu.h b/include/rp_kernel_on_cpu.h index 5b89f62..a0fda1c 100644 --- a/include/rp_kernel_on_cpu.h +++ b/include/rp_kernel_on_cpu.h @@ -9,7 +9,7 @@ #define swap_workaround(n) __builtin_bswap32(n) #include "common.h" -#include "rp_kernel.h" +#include "inc_rp.h" u32 apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len); u32 apply_rules (u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len); diff --git a/include/shared.h b/include/shared.h index d5b00ab..5b64b5b 100644 --- a/include/shared.h +++ b/include/shared.h @@ -9,8 +9,8 @@ #ifndef SHARED_H #define SHARED_H -#include -#include +#include "common.h" +#include "inc_hash_constants.h" /** * thread management @@ -108,18 +108,18 @@ static inline int CPU_ISSET (int num, cpu_set_t *cs) { return (cs->count & (1 < #define hc_sleep(x) sleep ((x)); #endif -#include +#include "ext_OpenCL.h" /** * temperature management */ #if _WIN -#include -#include +#include "ext_ADL.h" +#include "ext_nvapi.h" #else -#include -#include +#include "ext_ADL.h" +#include "ext_nvml.h" #endif /** @@ -184,9 +184,9 @@ typedef pthread_t hc_thread_t; typedef pthread_mutex_t hc_thread_mutex_t; #endif -#include +#include "types.h" #include "rp_cpu.h" -#include "rp_kernel.h" +#include "inc_rp.h" /** * valid project specific global stuff diff --git a/src/Makefile b/src/Makefile index 7509900..1515e5a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -87,7 +87,7 @@ VERSION_TAG := $(shell test -d .git && git describe --tags --dirty= ## Compiler flags ## -CFLAGS := -pipe -W -Wall -std=c99 -Iinclude/ -I$(OPENCL_HEADERS_KHRONOS)/ +CFLAGS := -pipe -W -Wall -std=c99 -Iinclude/ -IOpenCL/ -I$(OPENCL_HEADERS_KHRONOS)/ ifndef DEBUG CFLAGS += -O2 diff --git a/src/hashcat.c b/src/hashcat.c index e23aabd..417b60d 100644 --- a/src/hashcat.c +++ b/src/hashcat.c @@ -14556,7 +14556,7 @@ int main (int argc, char **argv) // we don't have sm_* on vendors not NV but it doesn't matter - snprintf (build_opts, sizeof (build_opts) - 1, "-cl-std=CL1.1 -I\"%s/\" -DVENDOR_ID=%u -DCUDA_ARCH=%d -DVECT_SIZE=%u -DDEVICE_TYPE=%u -DKERN_TYPE=%u -D_unroll", shared_dir, device_param->vendor_id, (device_param->sm_major * 100) + device_param->sm_minor, device_param->vector_width, (u32) device_param->device_type, kern_type); + snprintf (build_opts, sizeof (build_opts) - 1, "-cl-std=CL1.1 -I\"%s/OpenCL\" -DVENDOR_ID=%u -DCUDA_ARCH=%d -DVECT_SIZE=%u -DDEVICE_TYPE=%u -DKERN_TYPE=%u -D_unroll", shared_dir, device_param->vendor_id, (device_param->sm_major * 100) + device_param->sm_minor, device_param->vector_width, (u32) device_param->device_type, kern_type); if (device_param->vendor_id == VENDOR_ID_INTEL_SDK) { diff --git a/tools/package_bin.sh b/tools/package_bin.sh index f0220f1..67328a9 100755 --- a/tools/package_bin.sh +++ b/tools/package_bin.sh @@ -12,7 +12,6 @@ rm -rf $OUT rm -rf $OUT.7z mkdir -p $OUT -mkdir -p $OUT/include cp $IN/hashcat??.exe $OUT/ cp $IN/hashcat??.bin $OUT/ @@ -27,11 +26,6 @@ cp -r $IN/extra $OUT/ cp $IN/example.dict $OUT/ cp $IN/example[0123456789]*.hash $OUT/ cp $IN/example[0123456789]*.cmd $OUT/ - -cp -r $IN/include/constants.h $OUT/include -cp -r $IN/include/kernel_functions.c $OUT/include -cp -r $IN/include/kernel_vendor.h $OUT/include -cp -r $IN/include/rp_kernel.h $OUT/include cp -r $IN/OpenCL $OUT/ # since for the binary distribution we still use .bin, we need to rewrite the commands @@ -54,7 +48,6 @@ unix2dos $OUT/rules/*.rule unix2dos $OUT/rules/hybrid/*.rule unix2dos $OUT/docs/* unix2dos $OUT/example*.cmd -unix2dos $OUT/include/* unix2dos $OUT/OpenCL/* unix2dos $OUT/hashcat.hctune @@ -72,7 +65,6 @@ chmod 755 $OUT/example*.sh chmod 755 $OUT/extra chmod 755 $OUT/extra/tab_completion/*.sh chmod 755 $OUT/extra/tab_completion/install -chmod 755 $OUT/include chmod 644 $OUT/include/* chmod 755 $OUT/OpenCL chmod 644 $OUT/OpenCL/*