Move files from include/ to OpenCL/ if they are used within kernels

Rename includes in OpenCL so that it's easier to recognize them as such

diff --git a/OpenCL/amp_a0.cl b/OpenCL/amp_a0.cl
index dbc4ad8..821c7ce 100644 (file)
--- a/OpenCL/amp_a0.cl
+++ b/OpenCL/amp_a0.cl
@@ -3,12 +3,12 @@
  * License.....: MIT
  */
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
-#include "OpenCL/types_ocl.c"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
+#include "inc_types.cl"
 
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
+#include "inc_rp.h"
+#include "inc_rp.cl"
 
 __kernel void amp (__global pw_t *pws, __global pw_t *pws_amp, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/amp_a1.cl b/OpenCL/amp_a1.cl
index 553b1f6..d18e37c 100644 (file)
--- a/OpenCL/amp_a1.cl
+++ b/OpenCL/amp_a1.cl
@@ -3,9 +3,9 @@
  * License.....: MIT
  */
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
-#include "OpenCL/types_ocl.c"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
+#include "inc_types.cl"
 
 inline void switch_buffer_by_offset_le (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset)
 {

diff --git a/OpenCL/amp_a3.cl b/OpenCL/amp_a3.cl
index 370958b..f1b1b04 100644 (file)
--- a/OpenCL/amp_a3.cl
+++ b/OpenCL/amp_a3.cl
@@ -3,9 +3,9 @@
  * License.....: MIT
  */
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
-#include "OpenCL/types_ocl.c"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
+#include "inc_types.cl"
 
 __kernel void amp (__global pw_t *pws, __global pw_t *pws_amp, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/check_multi_comp4.c b/OpenCL/check_multi_comp4.c
deleted file mode 100644 (file)
index 14ad55c..0000000
--- a/OpenCL/check_multi_comp4.c
+++ /dev/null
@@ -1,32 +0,0 @@
-u32 digest_tp[4];
-
-digest_tp[0] = r0;
-digest_tp[1] = r1;
-digest_tp[2] = r2;
-digest_tp[3] = r3;
-
-if (check (digest_tp,
-             bitmaps_buf_s1_a,
-             bitmaps_buf_s1_b,
-             bitmaps_buf_s1_c,
-             bitmaps_buf_s1_d,
-             bitmaps_buf_s2_a,
-             bitmaps_buf_s2_b,
-             bitmaps_buf_s2_c,
-             bitmaps_buf_s2_d,
-             bitmap_mask,
-             bitmap_shift1,
-             bitmap_shift2))
-{
-  int digest_pos = find_hash (digest_tp, digests_cnt, &digests_buf[digests_offset]);
-
-  if (digest_pos != -1)
-  {
-    const u32 final_hash_pos = digests_offset + digest_pos;
-
-    if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)
-    {
-      mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos);
-    }
-  }
-}

diff --git a/OpenCL/check_multi_comp4_bs.c b/OpenCL/check_multi_comp4_bs.c
deleted file mode 100644 (file)
index 6546a97..0000000
--- a/OpenCL/check_multi_comp4_bs.c
+++ /dev/null
@@ -1,32 +0,0 @@
-u32 digest_tp[4];
-
-digest_tp[0] = r0;
-digest_tp[1] = r1;
-digest_tp[2] = r2;
-digest_tp[3] = r3;
-
-if (check (digest_tp,
-             bitmaps_buf_s1_a,
-             bitmaps_buf_s1_b,
-             bitmaps_buf_s1_c,
-             bitmaps_buf_s1_d,
-             bitmaps_buf_s2_a,
-             bitmaps_buf_s2_b,
-             bitmaps_buf_s2_c,
-             bitmaps_buf_s2_d,
-             bitmap_mask,
-             bitmap_shift1,
-             bitmap_shift2))
-{
-  int digest_pos = find_hash (digest_tp, digests_cnt, &digests_buf[digests_offset]);
-
-  if (digest_pos != -1)
-  {
-    const u32 final_hash_pos = digests_offset + digest_pos;
-
-    if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)
-    {
-      mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + slice);
-    }
-  }
-}

diff --git a/OpenCL/check_single_comp4.c b/OpenCL/check_single_comp4.c
deleted file mode 100644 (file)
index 631a8bc..0000000
--- a/OpenCL/check_single_comp4.c
+++ /dev/null
@@ -1,12 +0,0 @@
-if ((r0 == search[0])
- && (r1 == search[1])
- && (r2 == search[2])
- && (r3 == search[3]))
-{
-  const u32 final_hash_pos = digests_offset + 0;
-
-  if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)
-  {
-    mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos);
-  }
-}

diff --git a/OpenCL/check_single_comp4_bs.c b/OpenCL/check_single_comp4_bs.c
deleted file mode 100644 (file)
index 5c081c1..0000000
--- a/OpenCL/check_single_comp4_bs.c
+++ /dev/null
@@ -1,6 +0,0 @@
-const u32 final_hash_pos = digests_offset + 0;
-
-if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)
-{
-  mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos);
-}

diff --git a/OpenCL/common.c b/OpenCL/common.c
deleted file mode 100644 (file)
index ca811e5..0000000
--- a/OpenCL/common.c
+++ /dev/null
@@ -1,9324 +0,0 @@
-/**
- * Author......: Jens Steube <jens.steube@gmail.com>
- * License.....: MIT
- */
-
-/**
- * pure scalar functions
- */
-
-inline int hash_comp (const u32 d1[4], __global u32 *d2)
-{
-  if (d1[3] > d2[DGST_R3]) return ( 1);
-  if (d1[3] < d2[DGST_R3]) return (-1);
-  if (d1[2] > d2[DGST_R2]) return ( 1);
-  if (d1[2] < d2[DGST_R2]) return (-1);
-  if (d1[1] > d2[DGST_R1]) return ( 1);
-  if (d1[1] < d2[DGST_R1]) return (-1);
-  if (d1[0] > d2[DGST_R0]) return ( 1);
-  if (d1[0] < d2[DGST_R0]) return (-1);
-
-  return (0);
-}
-
-inline int find_hash (const u32 digest[4], const u32 digests_cnt, __global digest_t *digests_buf)
-{
-  for (u32 l = 0, r = digests_cnt; r; r >>= 1)
-  {
-    const u32 m = r >> 1;
-
-    const u32 c = l + m;
-
-    const int cmp = hash_comp (digest, digests_buf[c].digest_buf);
-
-    if (cmp > 0)
-    {
-      l += m + 1;
-
-      r--;
-    }
-
-    if (cmp == 0) return (c);
-  }
-
-  return (-1);
-}
-
-inline u32 check_bitmap (__global u32 *bitmap, const u32 bitmap_mask, const u32 bitmap_shift, const u32 digest)
-{
-  return (bitmap[(digest >> bitmap_shift) & bitmap_mask] & (1 << (digest & 0x1f)));
-}
-
-inline u32 check (const u32 digest[2], __global u32 *bitmap_s1_a, __global u32 *bitmap_s1_b, __global u32 *bitmap_s1_c, __global u32 *bitmap_s1_d, __global u32 *bitmap_s2_a, __global u32 *bitmap_s2_b, __global u32 *bitmap_s2_c, __global u32 *bitmap_s2_d, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2)
-{
-  if (check_bitmap (bitmap_s1_a, bitmap_mask, bitmap_shift1, digest[0]) == 0) return (0);
-  if (check_bitmap (bitmap_s1_b, bitmap_mask, bitmap_shift1, digest[1]) == 0) return (0);
-  if (check_bitmap (bitmap_s1_c, bitmap_mask, bitmap_shift1, digest[2]) == 0) return (0);
-  if (check_bitmap (bitmap_s1_d, bitmap_mask, bitmap_shift1, digest[3]) == 0) return (0);
-
-  if (check_bitmap (bitmap_s2_a, bitmap_mask, bitmap_shift2, digest[0]) == 0) return (0);
-  if (check_bitmap (bitmap_s2_b, bitmap_mask, bitmap_shift2, digest[1]) == 0) return (0);
-  if (check_bitmap (bitmap_s2_c, bitmap_mask, bitmap_shift2, digest[2]) == 0) return (0);
-  if (check_bitmap (bitmap_s2_d, bitmap_mask, bitmap_shift2, digest[3]) == 0) return (0);
-
-  return (1);
-}
-
-inline void mark_hash (__global plain_t *plains_buf, __global u32 *d_result, const int salt_pos, const int digest_pos, const int hash_pos, const u32 gid, const u32 il_pos)
-{
-  const u32 idx = atomic_inc (d_result);
-
-  plains_buf[idx].salt_pos    = salt_pos;
-  plains_buf[idx].digest_pos  = digest_pos; // relative
-  plains_buf[idx].hash_pos    = hash_pos;   // absolute
-  plains_buf[idx].gidvid      = gid;
-  plains_buf[idx].il_pos      = il_pos;
-}
-
-/**
- * vector functions
- */
-
-inline void truncate_block (u32x w[4], const u32 len)
-{
-  switch (len)
-  {
-    case  0:  w[0] &= 0;
-              w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  1:  w[0] &= 0x000000FF;
-              w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  2:  w[0] &= 0x0000FFFF;
-              w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  3:  w[0] &= 0x00FFFFFF;
-              w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  4:  w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  5:  w[1] &= 0x000000FF;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  6:  w[1] &= 0x0000FFFF;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  7:  w[1] &= 0x00FFFFFF;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  8:  w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  9:  w[2] &= 0x000000FF;
-              w[3] &= 0;
-              break;
-    case 10:  w[2] &= 0x0000FFFF;
-              w[3] &= 0;
-              break;
-    case 11:  w[2] &= 0x00FFFFFF;
-              w[3] &= 0;
-              break;
-    case 12:  w[3] &= 0;
-              break;
-    case 13:  w[3] &= 0x000000FF;
-              break;
-    case 14:  w[3] &= 0x0000FFFF;
-              break;
-    case 15:  w[3] &= 0x00FFFFFF;
-              break;
-  }
-}
-
-inline void make_unicode (const u32x in[4], u32x out1[4], u32x out2[4])
-{
-  #ifdef IS_NV
-  out2[3] = __byte_perm (in[3], 0, 0x7372);
-  out2[2] = __byte_perm (in[3], 0, 0x7170);
-  out2[1] = __byte_perm (in[2], 0, 0x7372);
-  out2[0] = __byte_perm (in[2], 0, 0x7170);
-  out1[3] = __byte_perm (in[1], 0, 0x7372);
-  out1[2] = __byte_perm (in[1], 0, 0x7170);
-  out1[1] = __byte_perm (in[0], 0, 0x7372);
-  out1[0] = __byte_perm (in[0], 0, 0x7170);
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  out2[3]  = ((in[3] >> 8) & 0x00FF0000) | ((in[3] >> 16) & 0x000000FF);
-  out2[2]  = ((in[3] << 8) & 0x00FF0000) | ((in[3] >>  0) & 0x000000FF);
-  out2[1]  = ((in[2] >> 8) & 0x00FF0000) | ((in[2] >> 16) & 0x000000FF);
-  out2[0]  = ((in[2] << 8) & 0x00FF0000) | ((in[2] >>  0) & 0x000000FF);
-  out1[3]  = ((in[1] >> 8) & 0x00FF0000) | ((in[1] >> 16) & 0x000000FF);
-  out1[2]  = ((in[1] << 8) & 0x00FF0000) | ((in[1] >>  0) & 0x000000FF);
-  out1[1]  = ((in[0] >> 8) & 0x00FF0000) | ((in[0] >> 16) & 0x000000FF);
-  out1[0]  = ((in[0] << 8) & 0x00FF0000) | ((in[0] >>  0) & 0x000000FF);
-  #endif
-}
-
-inline void undo_unicode (const u32x in1[4], const u32x in2[4], u32x out[4])
-{
-  #ifdef IS_NV
-  out[0] = __byte_perm (in1[0], in1[1], 0x6420);
-  out[1] = __byte_perm (in1[2], in1[3], 0x6420);
-  out[2] = __byte_perm (in2[0], in2[1], 0x6420);
-  out[3] = __byte_perm (in2[2], in2[3], 0x6420);
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  out[0] = ((in1[0] & 0x000000ff) >>  0) | ((in1[0] & 0x00ff0000) >>  8)
-         | ((in1[1] & 0x000000ff) << 16) | ((in1[1] & 0x00ff0000) <<  8);
-  out[1] = ((in1[2] & 0x000000ff) >>  0) | ((in1[2] & 0x00ff0000) >>  8)
-         | ((in1[3] & 0x000000ff) << 16) | ((in1[3] & 0x00ff0000) <<  8);
-  out[2] = ((in2[0] & 0x000000ff) >>  0) | ((in2[0] & 0x00ff0000) >>  8)
-         | ((in2[1] & 0x000000ff) << 16) | ((in2[1] & 0x00ff0000) <<  8);
-  out[3] = ((in2[2] & 0x000000ff) >>  0) | ((in2[2] & 0x00ff0000) >>  8)
-         | ((in2[3] & 0x000000ff) << 16) | ((in2[3] & 0x00ff0000) <<  8);
-  #endif
-}
-
-inline void append_0x01_1x4 (u32x w0[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0]  = 0x01;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0100;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x010000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x01000000;
-      break;
-
-    case 4:
-      w0[1] = 0x01;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0100;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x010000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x01000000;
-      break;
-
-    case 8:
-      w0[2] = 0x01;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0100;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x010000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x01000000;
-      break;
-
-    case 12:
-      w0[3] = 0x01;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0100;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x010000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x01000000;
-      break;
-  }
-}
-
-inline void append_0x01_2x4 (u32x w0[4], u32x w1[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x01;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0100;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x010000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x01000000;
-      break;
-
-    case 4:
-      w0[1] = 0x01;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0100;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x010000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x01000000;
-      break;
-
-    case 8:
-      w0[2] = 0x01;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0100;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x010000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x01000000;
-      break;
-
-    case 12:
-      w0[3] = 0x01;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0100;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x010000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x01000000;
-      break;
-
-    case 16:
-      w1[0] = 0x01;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0100;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x010000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x01000000;
-      break;
-
-    case 20:
-      w1[1] = 0x01;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0100;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x010000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x01000000;
-      break;
-
-    case 24:
-      w1[2] = 0x01;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0100;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x010000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x01000000;
-      break;
-
-    case 28:
-      w1[3] = 0x01;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0100;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x010000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x01000000;
-      break;
-  }
-}
-
-inline void append_0x01_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x01;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0100;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x010000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x01000000;
-      break;
-
-    case 4:
-      w0[1] = 0x01;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0100;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x010000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x01000000;
-      break;
-
-    case 8:
-      w0[2] = 0x01;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0100;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x010000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x01000000;
-      break;
-
-    case 12:
-      w0[3] = 0x01;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0100;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x010000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x01000000;
-      break;
-
-    case 16:
-      w1[0] = 0x01;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0100;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x010000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x01000000;
-      break;
-
-    case 20:
-      w1[1] = 0x01;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0100;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x010000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x01000000;
-      break;
-
-    case 24:
-      w1[2] = 0x01;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0100;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x010000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x01000000;
-      break;
-
-    case 28:
-      w1[3] = 0x01;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0100;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x010000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x01000000;
-      break;
-
-    case 32:
-      w2[0] = 0x01;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x0100;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x010000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x01000000;
-      break;
-
-    case 36:
-      w2[1] = 0x01;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x0100;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x010000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x01000000;
-      break;
-
-    case 40:
-      w2[2] = 0x01;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x0100;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x010000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x01000000;
-      break;
-
-    case 44:
-      w2[3] = 0x01;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x0100;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x010000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x01000000;
-      break;
-  }
-}
-
-inline void append_0x01_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x01;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0100;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x010000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x01000000;
-      break;
-
-    case 4:
-      w0[1] = 0x01;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0100;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x010000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x01000000;
-      break;
-
-    case 8:
-      w0[2] = 0x01;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0100;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x010000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x01000000;
-      break;
-
-    case 12:
-      w0[3] = 0x01;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0100;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x010000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x01000000;
-      break;
-
-    case 16:
-      w1[0] = 0x01;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0100;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x010000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x01000000;
-      break;
-
-    case 20:
-      w1[1] = 0x01;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0100;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x010000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x01000000;
-      break;
-
-    case 24:
-      w1[2] = 0x01;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0100;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x010000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x01000000;
-      break;
-
-    case 28:
-      w1[3] = 0x01;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0100;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x010000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x01000000;
-      break;
-
-    case 32:
-      w2[0] = 0x01;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x0100;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x010000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x01000000;
-      break;
-
-    case 36:
-      w2[1] = 0x01;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x0100;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x010000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x01000000;
-      break;
-
-    case 40:
-      w2[2] = 0x01;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x0100;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x010000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x01000000;
-      break;
-
-    case 44:
-      w2[3] = 0x01;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x0100;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x010000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x01000000;
-      break;
-
-    case 48:
-      w3[0] = 0x01;
-      break;
-
-    case 49:
-      w3[0] = w3[0] | 0x0100;
-      break;
-
-    case 50:
-      w3[0] = w3[0] | 0x010000;
-      break;
-
-    case 51:
-      w3[0] = w3[0] | 0x01000000;
-      break;
-
-    case 52:
-      w3[1] = 0x01;
-      break;
-
-    case 53:
-      w3[1] = w3[1] | 0x0100;
-      break;
-
-    case 54:
-      w3[1] = w3[1] | 0x010000;
-      break;
-
-    case 55:
-      w3[1] = w3[1] | 0x01000000;
-      break;
-
-    case 56:
-      w3[2] = 0x01;
-      break;
-
-    case 57:
-      w3[2] = w3[2] | 0x0100;
-      break;
-
-    case 58:
-      w3[2] = w3[2] | 0x010000;
-      break;
-
-    case 59:
-      w3[2] = w3[2] | 0x01000000;
-      break;
-
-    case 60:
-      w3[3] = 0x01;
-      break;
-
-    case 61:
-      w3[3] = w3[3] | 0x0100;
-      break;
-
-    case 62:
-      w3[3] = w3[3] | 0x010000;
-      break;
-
-    case 63:
-      w3[3] = w3[3] | 0x01000000;
-      break;
-  }
-}
-
-inline void append_0x01_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x01;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0100;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x010000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x01000000;
-      break;
-
-    case 4:
-      w0[1] = 0x01;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0100;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x010000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x01000000;
-      break;
-
-    case 8:
-      w0[2] = 0x01;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0100;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x010000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x01000000;
-      break;
-
-    case 12:
-      w0[3] = 0x01;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0100;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x010000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x01000000;
-      break;
-
-    case 16:
-      w1[0] = 0x01;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0100;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x010000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x01000000;
-      break;
-
-    case 20:
-      w1[1] = 0x01;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0100;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x010000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x01000000;
-      break;
-
-    case 24:
-      w1[2] = 0x01;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0100;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x010000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x01000000;
-      break;
-
-    case 28:
-      w1[3] = 0x01;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0100;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x010000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x01000000;
-      break;
-
-    case 32:
-      w2[0] = 0x01;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x0100;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x010000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x01000000;
-      break;
-
-    case 36:
-      w2[1] = 0x01;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x0100;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x010000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x01000000;
-      break;
-
-    case 40:
-      w2[2] = 0x01;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x0100;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x010000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x01000000;
-      break;
-
-    case 44:
-      w2[3] = 0x01;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x0100;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x010000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x01000000;
-      break;
-
-    case 48:
-      w3[0] = 0x01;
-      break;
-
-    case 49:
-      w3[0] = w3[0] | 0x0100;
-      break;
-
-    case 50:
-      w3[0] = w3[0] | 0x010000;
-      break;
-
-    case 51:
-      w3[0] = w3[0] | 0x01000000;
-      break;
-
-    case 52:
-      w3[1] = 0x01;
-      break;
-
-    case 53:
-      w3[1] = w3[1] | 0x0100;
-      break;
-
-    case 54:
-      w3[1] = w3[1] | 0x010000;
-      break;
-
-    case 55:
-      w3[1] = w3[1] | 0x01000000;
-      break;
-
-    case 56:
-      w3[2] = 0x01;
-      break;
-
-    case 57:
-      w3[2] = w3[2] | 0x0100;
-      break;
-
-    case 58:
-      w3[2] = w3[2] | 0x010000;
-      break;
-
-    case 59:
-      w3[2] = w3[2] | 0x01000000;
-      break;
-
-    case 60:
-      w3[3] = 0x01;
-      break;
-
-    case 61:
-      w3[3] = w3[3] | 0x0100;
-      break;
-
-    case 62:
-      w3[3] = w3[3] | 0x010000;
-      break;
-
-    case 63:
-      w3[3] = w3[3] | 0x01000000;
-      break;
-
-    case 64:
-      w4[0] = 0x01;
-      break;
-
-    case 65:
-      w4[0] = w4[0] | 0x0100;
-      break;
-
-    case 66:
-      w4[0] = w4[0] | 0x010000;
-      break;
-
-    case 67:
-      w4[0] = w4[0] | 0x01000000;
-      break;
-
-    case 68:
-      w4[1] = 0x01;
-      break;
-
-    case 69:
-      w4[1] = w4[1] | 0x0100;
-      break;
-
-    case 70:
-      w4[1] = w4[1] | 0x010000;
-      break;
-
-    case 71:
-      w4[1] = w4[1] | 0x01000000;
-      break;
-
-    case 72:
-      w4[2] = 0x01;
-      break;
-
-    case 73:
-      w4[2] = w4[2] | 0x0100;
-      break;
-
-    case 74:
-      w4[2] = w4[2] | 0x010000;
-      break;
-
-    case 75:
-      w4[2] = w4[2] | 0x01000000;
-      break;
-
-    case 76:
-      w4[3] = 0x01;
-      break;
-
-    case 77:
-      w4[3] = w4[3] | 0x0100;
-      break;
-
-    case 78:
-      w4[3] = w4[3] | 0x010000;
-      break;
-
-    case 79:
-      w4[3] = w4[3] | 0x01000000;
-      break;
-
-    case 80:
-      w5[0] = 0x01;
-      break;
-
-    case 81:
-      w5[0] = w5[0] | 0x0100;
-      break;
-
-    case 82:
-      w5[0] = w5[0] | 0x010000;
-      break;
-
-    case 83:
-      w5[0] = w5[0] | 0x01000000;
-      break;
-
-    case 84:
-      w5[1] = 0x01;
-      break;
-
-    case 85:
-      w5[1] = w5[1] | 0x0100;
-      break;
-
-    case 86:
-      w5[1] = w5[1] | 0x010000;
-      break;
-
-    case 87:
-      w5[1] = w5[1] | 0x01000000;
-      break;
-
-    case 88:
-      w5[2] = 0x01;
-      break;
-
-    case 89:
-      w5[2] = w5[2] | 0x0100;
-      break;
-
-    case 90:
-      w5[2] = w5[2] | 0x010000;
-      break;
-
-    case 91:
-      w5[2] = w5[2] | 0x01000000;
-      break;
-
-    case 92:
-      w5[3] = 0x01;
-      break;
-
-    case 93:
-      w5[3] = w5[3] | 0x0100;
-      break;
-
-    case 94:
-      w5[3] = w5[3] | 0x010000;
-      break;
-
-    case 95:
-      w5[3] = w5[3] | 0x01000000;
-      break;
-
-    case 96:
-      w6[0] = 0x01;
-      break;
-
-    case 97:
-      w6[0] = w6[0] | 0x0100;
-      break;
-
-    case 98:
-      w6[0] = w6[0] | 0x010000;
-      break;
-
-    case 99:
-      w6[0] = w6[0] | 0x01000000;
-      break;
-
-    case 100:
-      w6[1] = 0x01;
-      break;
-
-    case 101:
-      w6[1] = w6[1] | 0x0100;
-      break;
-
-    case 102:
-      w6[1] = w6[1] | 0x010000;
-      break;
-
-    case 103:
-      w6[1] = w6[1] | 0x01000000;
-      break;
-
-    case 104:
-      w6[2] = 0x01;
-      break;
-
-    case 105:
-      w6[2] = w6[2] | 0x0100;
-      break;
-
-    case 106:
-      w6[2] = w6[2] | 0x010000;
-      break;
-
-    case 107:
-      w6[2] = w6[2] | 0x01000000;
-      break;
-
-    case 108:
-      w6[3] = 0x01;
-      break;
-
-    case 109:
-      w6[3] = w6[3] | 0x0100;
-      break;
-
-    case 110:
-      w6[3] = w6[3] | 0x010000;
-      break;
-
-    case 111:
-      w6[3] = w6[3] | 0x01000000;
-      break;
-
-    case 112:
-      w7[0] = 0x01;
-      break;
-
-    case 113:
-      w7[0] = w7[0] | 0x0100;
-      break;
-
-    case 114:
-      w7[0] = w7[0] | 0x010000;
-      break;
-
-    case 115:
-      w7[0] = w7[0] | 0x01000000;
-      break;
-
-    case 116:
-      w7[1] = 0x01;
-      break;
-
-    case 117:
-      w7[1] = w7[1] | 0x0100;
-      break;
-
-    case 118:
-      w7[1] = w7[1] | 0x010000;
-      break;
-
-    case 119:
-      w7[1] = w7[1] | 0x01000000;
-      break;
-
-    case 120:
-      w7[2] = 0x01;
-      break;
-
-    case 121:
-      w7[2] = w7[2] | 0x0100;
-      break;
-
-    case 122:
-      w7[2] = w7[2] | 0x010000;
-      break;
-
-    case 123:
-      w7[2] = w7[2] | 0x01000000;
-      break;
-
-    case 124:
-      w7[3] = 0x01;
-      break;
-
-    case 125:
-      w7[3] = w7[3] | 0x0100;
-      break;
-
-    case 126:
-      w7[3] = w7[3] | 0x010000;
-      break;
-
-    case 127:
-      w7[3] = w7[3] | 0x01000000;
-      break;
-  }
-}
-
-inline void append_0x02_1x4 (u32x w0[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0]  = 0x02;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0200;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x020000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x02000000;
-      break;
-
-    case 4:
-      w0[1] = 0x02;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0200;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x020000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x02000000;
-      break;
-
-    case 8:
-      w0[2] = 0x02;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0200;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x020000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x02000000;
-      break;
-
-    case 12:
-      w0[3] = 0x02;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0200;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x020000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x02000000;
-      break;
-  }
-}
-
-inline void append_0x02_2x4 (u32x w0[4], u32x w1[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x02;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0200;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x020000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x02000000;
-      break;
-
-    case 4:
-      w0[1] = 0x02;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0200;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x020000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x02000000;
-      break;
-
-    case 8:
-      w0[2] = 0x02;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0200;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x020000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x02000000;
-      break;
-
-    case 12:
-      w0[3] = 0x02;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0200;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x020000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x02000000;
-      break;
-
-    case 16:
-      w1[0] = 0x02;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0200;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x020000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x02000000;
-      break;
-
-    case 20:
-      w1[1] = 0x02;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0200;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x020000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x02000000;
-      break;
-
-    case 24:
-      w1[2] = 0x02;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0200;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x020000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x02000000;
-      break;
-
-    case 28:
-      w1[3] = 0x02;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0200;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x020000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x02000000;
-      break;
-  }
-}
-
-inline void append_0x02_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x02;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0200;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x020000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x02000000;
-      break;
-
-    case 4:
-      w0[1] = 0x02;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0200;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x020000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x02000000;
-      break;
-
-    case 8:
-      w0[2] = 0x02;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0200;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x020000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x02000000;
-      break;
-
-    case 12:
-      w0[3] = 0x02;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0200;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x020000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x02000000;
-      break;
-
-    case 16:
-      w1[0] = 0x02;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0200;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x020000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x02000000;
-      break;
-
-    case 20:
-      w1[1] = 0x02;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0200;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x020000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x02000000;
-      break;
-
-    case 24:
-      w1[2] = 0x02;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0200;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x020000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x02000000;
-      break;
-
-    case 28:
-      w1[3] = 0x02;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0200;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x020000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x02000000;
-      break;
-
-    case 32:
-      w2[0] = 0x02;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x0200;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x020000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x02000000;
-      break;
-
-    case 36:
-      w2[1] = 0x02;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x0200;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x020000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x02000000;
-      break;
-
-    case 40:
-      w2[2] = 0x02;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x0200;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x020000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x02000000;
-      break;
-
-    case 44:
-      w2[3] = 0x02;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x0200;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x020000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x02000000;
-      break;
-  }
-}
-
-inline void append_0x02_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x02;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0200;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x020000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x02000000;
-      break;
-
-    case 4:
-      w0[1] = 0x02;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0200;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x020000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x02000000;
-      break;
-
-    case 8:
-      w0[2] = 0x02;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0200;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x020000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x02000000;
-      break;
-
-    case 12:
-      w0[3] = 0x02;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0200;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x020000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x02000000;
-      break;
-
-    case 16:
-      w1[0] = 0x02;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0200;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x020000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x02000000;
-      break;
-
-    case 20:
-      w1[1] = 0x02;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0200;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x020000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x02000000;
-      break;
-
-    case 24:
-      w1[2] = 0x02;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0200;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x020000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x02000000;
-      break;
-
-    case 28:
-      w1[3] = 0x02;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0200;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x020000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x02000000;
-      break;
-
-    case 32:
-      w2[0] = 0x02;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x0200;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x020000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x02000000;
-      break;
-
-    case 36:
-      w2[1] = 0x02;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x0200;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x020000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x02000000;
-      break;
-
-    case 40:
-      w2[2] = 0x02;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x0200;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x020000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x02000000;
-      break;
-
-    case 44:
-      w2[3] = 0x02;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x0200;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x020000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x02000000;
-      break;
-
-    case 48:
-      w3[0] = 0x02;
-      break;
-
-    case 49:
-      w3[0] = w3[0] | 0x0200;
-      break;
-
-    case 50:
-      w3[0] = w3[0] | 0x020000;
-      break;
-
-    case 51:
-      w3[0] = w3[0] | 0x02000000;
-      break;
-
-    case 52:
-      w3[1] = 0x02;
-      break;
-
-    case 53:
-      w3[1] = w3[1] | 0x0200;
-      break;
-
-    case 54:
-      w3[1] = w3[1] | 0x020000;
-      break;
-
-    case 55:
-      w3[1] = w3[1] | 0x02000000;
-      break;
-
-    case 56:
-      w3[2] = 0x02;
-      break;
-
-    case 57:
-      w3[2] = w3[2] | 0x0200;
-      break;
-
-    case 58:
-      w3[2] = w3[2] | 0x020000;
-      break;
-
-    case 59:
-      w3[2] = w3[2] | 0x02000000;
-      break;
-
-    case 60:
-      w3[3] = 0x02;
-      break;
-
-    case 61:
-      w3[3] = w3[3] | 0x0200;
-      break;
-
-    case 62:
-      w3[3] = w3[3] | 0x020000;
-      break;
-
-    case 63:
-      w3[3] = w3[3] | 0x02000000;
-      break;
-  }
-}
-
-inline void append_0x02_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x02;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0200;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x020000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x02000000;
-      break;
-
-    case 4:
-      w0[1] = 0x02;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0200;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x020000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x02000000;
-      break;
-
-    case 8:
-      w0[2] = 0x02;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0200;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x020000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x02000000;
-      break;
-
-    case 12:
-      w0[3] = 0x02;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0200;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x020000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x02000000;
-      break;
-
-    case 16:
-      w1[0] = 0x02;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0200;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x020000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x02000000;
-      break;
-
-    case 20:
-      w1[1] = 0x02;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0200;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x020000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x02000000;
-      break;
-
-    case 24:
-      w1[2] = 0x02;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0200;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x020000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x02000000;
-      break;
-
-    case 28:
-      w1[3] = 0x02;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0200;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x020000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x02000000;
-      break;
-
-    case 32:
-      w2[0] = 0x02;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x0200;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x020000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x02000000;
-      break;
-
-    case 36:
-      w2[1] = 0x02;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x0200;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x020000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x02000000;
-      break;
-
-    case 40:
-      w2[2] = 0x02;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x0200;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x020000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x02000000;
-      break;
-
-    case 44:
-      w2[3] = 0x02;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x0200;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x020000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x02000000;
-      break;
-
-    case 48:
-      w3[0] = 0x02;
-      break;
-
-    case 49:
-      w3[0] = w3[0] | 0x0200;
-      break;
-
-    case 50:
-      w3[0] = w3[0] | 0x020000;
-      break;
-
-    case 51:
-      w3[0] = w3[0] | 0x02000000;
-      break;
-
-    case 52:
-      w3[1] = 0x02;
-      break;
-
-    case 53:
-      w3[1] = w3[1] | 0x0200;
-      break;
-
-    case 54:
-      w3[1] = w3[1] | 0x020000;
-      break;
-
-    case 55:
-      w3[1] = w3[1] | 0x02000000;
-      break;
-
-    case 56:
-      w3[2] = 0x02;
-      break;
-
-    case 57:
-      w3[2] = w3[2] | 0x0200;
-      break;
-
-    case 58:
-      w3[2] = w3[2] | 0x020000;
-      break;
-
-    case 59:
-      w3[2] = w3[2] | 0x02000000;
-      break;
-
-    case 60:
-      w3[3] = 0x02;
-      break;
-
-    case 61:
-      w3[3] = w3[3] | 0x0200;
-      break;
-
-    case 62:
-      w3[3] = w3[3] | 0x020000;
-      break;
-
-    case 63:
-      w3[3] = w3[3] | 0x02000000;
-      break;
-
-    case 64:
-      w4[0] = 0x02;
-      break;
-
-    case 65:
-      w4[0] = w4[0] | 0x0200;
-      break;
-
-    case 66:
-      w4[0] = w4[0] | 0x020000;
-      break;
-
-    case 67:
-      w4[0] = w4[0] | 0x02000000;
-      break;
-
-    case 68:
-      w4[1] = 0x02;
-      break;
-
-    case 69:
-      w4[1] = w4[1] | 0x0200;
-      break;
-
-    case 70:
-      w4[1] = w4[1] | 0x020000;
-      break;
-
-    case 71:
-      w4[1] = w4[1] | 0x02000000;
-      break;
-
-    case 72:
-      w4[2] = 0x02;
-      break;
-
-    case 73:
-      w4[2] = w4[2] | 0x0200;
-      break;
-
-    case 74:
-      w4[2] = w4[2] | 0x020000;
-      break;
-
-    case 75:
-      w4[2] = w4[2] | 0x02000000;
-      break;
-
-    case 76:
-      w4[3] = 0x02;
-      break;
-
-    case 77:
-      w4[3] = w4[3] | 0x0200;
-      break;
-
-    case 78:
-      w4[3] = w4[3] | 0x020000;
-      break;
-
-    case 79:
-      w4[3] = w4[3] | 0x02000000;
-      break;
-
-    case 80:
-      w5[0] = 0x02;
-      break;
-
-    case 81:
-      w5[0] = w5[0] | 0x0200;
-      break;
-
-    case 82:
-      w5[0] = w5[0] | 0x020000;
-      break;
-
-    case 83:
-      w5[0] = w5[0] | 0x02000000;
-      break;
-
-    case 84:
-      w5[1] = 0x02;
-      break;
-
-    case 85:
-      w5[1] = w5[1] | 0x0200;
-      break;
-
-    case 86:
-      w5[1] = w5[1] | 0x020000;
-      break;
-
-    case 87:
-      w5[1] = w5[1] | 0x02000000;
-      break;
-
-    case 88:
-      w5[2] = 0x02;
-      break;
-
-    case 89:
-      w5[2] = w5[2] | 0x0200;
-      break;
-
-    case 90:
-      w5[2] = w5[2] | 0x020000;
-      break;
-
-    case 91:
-      w5[2] = w5[2] | 0x02000000;
-      break;
-
-    case 92:
-      w5[3] = 0x02;
-      break;
-
-    case 93:
-      w5[3] = w5[3] | 0x0200;
-      break;
-
-    case 94:
-      w5[3] = w5[3] | 0x020000;
-      break;
-
-    case 95:
-      w5[3] = w5[3] | 0x02000000;
-      break;
-
-    case 96:
-      w6[0] = 0x02;
-      break;
-
-    case 97:
-      w6[0] = w6[0] | 0x0200;
-      break;
-
-    case 98:
-      w6[0] = w6[0] | 0x020000;
-      break;
-
-    case 99:
-      w6[0] = w6[0] | 0x02000000;
-      break;
-
-    case 100:
-      w6[1] = 0x02;
-      break;
-
-    case 101:
-      w6[1] = w6[1] | 0x0200;
-      break;
-
-    case 102:
-      w6[1] = w6[1] | 0x020000;
-      break;
-
-    case 103:
-      w6[1] = w6[1] | 0x02000000;
-      break;
-
-    case 104:
-      w6[2] = 0x02;
-      break;
-
-    case 105:
-      w6[2] = w6[2] | 0x0200;
-      break;
-
-    case 106:
-      w6[2] = w6[2] | 0x020000;
-      break;
-
-    case 107:
-      w6[2] = w6[2] | 0x02000000;
-      break;
-
-    case 108:
-      w6[3] = 0x02;
-      break;
-
-    case 109:
-      w6[3] = w6[3] | 0x0200;
-      break;
-
-    case 110:
-      w6[3] = w6[3] | 0x020000;
-      break;
-
-    case 111:
-      w6[3] = w6[3] | 0x02000000;
-      break;
-
-    case 112:
-      w7[0] = 0x02;
-      break;
-
-    case 113:
-      w7[0] = w7[0] | 0x0200;
-      break;
-
-    case 114:
-      w7[0] = w7[0] | 0x020000;
-      break;
-
-    case 115:
-      w7[0] = w7[0] | 0x02000000;
-      break;
-
-    case 116:
-      w7[1] = 0x02;
-      break;
-
-    case 117:
-      w7[1] = w7[1] | 0x0200;
-      break;
-
-    case 118:
-      w7[1] = w7[1] | 0x020000;
-      break;
-
-    case 119:
-      w7[1] = w7[1] | 0x02000000;
-      break;
-
-    case 120:
-      w7[2] = 0x02;
-      break;
-
-    case 121:
-      w7[2] = w7[2] | 0x0200;
-      break;
-
-    case 122:
-      w7[2] = w7[2] | 0x020000;
-      break;
-
-    case 123:
-      w7[2] = w7[2] | 0x02000000;
-      break;
-
-    case 124:
-      w7[3] = 0x02;
-      break;
-
-    case 125:
-      w7[3] = w7[3] | 0x0200;
-      break;
-
-    case 126:
-      w7[3] = w7[3] | 0x020000;
-      break;
-
-    case 127:
-      w7[3] = w7[3] | 0x02000000;
-      break;
-  }
-}
-
-inline void append_0x80_1x4 (u32x w0[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0]  = 0x80;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x8000;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x800000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x80000000;
-      break;
-
-    case 4:
-      w0[1] = 0x80;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x8000;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x800000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x80000000;
-      break;
-
-    case 8:
-      w0[2] = 0x80;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x8000;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x800000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x80000000;
-      break;
-
-    case 12:
-      w0[3] = 0x80;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x8000;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x800000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x80000000;
-      break;
-  }
-}
-
-inline void append_0x80_2x4 (u32x w0[4], u32x w1[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x80;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x8000;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x800000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x80000000;
-      break;
-
-    case 4:
-      w0[1] = 0x80;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x8000;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x800000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x80000000;
-      break;
-
-    case 8:
-      w0[2] = 0x80;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x8000;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x800000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x80000000;
-      break;
-
-    case 12:
-      w0[3] = 0x80;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x8000;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x800000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x80000000;
-      break;
-
-    case 16:
-      w1[0] = 0x80;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x8000;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x800000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x80000000;
-      break;
-
-    case 20:
-      w1[1] = 0x80;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x8000;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x800000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x80000000;
-      break;
-
-    case 24:
-      w1[2] = 0x80;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x8000;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x800000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x80000000;
-      break;
-
-    case 28:
-      w1[3] = 0x80;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x8000;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x800000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x80000000;
-      break;
-  }
-}
-
-inline void append_0x80_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x80;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x8000;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x800000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x80000000;
-      break;
-
-    case 4:
-      w0[1] = 0x80;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x8000;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x800000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x80000000;
-      break;
-
-    case 8:
-      w0[2] = 0x80;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x8000;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x800000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x80000000;
-      break;
-
-    case 12:
-      w0[3] = 0x80;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x8000;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x800000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x80000000;
-      break;
-
-    case 16:
-      w1[0] = 0x80;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x8000;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x800000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x80000000;
-      break;
-
-    case 20:
-      w1[1] = 0x80;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x8000;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x800000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x80000000;
-      break;
-
-    case 24:
-      w1[2] = 0x80;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x8000;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x800000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x80000000;
-      break;
-
-    case 28:
-      w1[3] = 0x80;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x8000;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x800000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x80000000;
-      break;
-
-    case 32:
-      w2[0] = 0x80;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x8000;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x800000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x80000000;
-      break;
-
-    case 36:
-      w2[1] = 0x80;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x8000;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x800000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x80000000;
-      break;
-
-    case 40:
-      w2[2] = 0x80;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x8000;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x800000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x80000000;
-      break;
-
-    case 44:
-      w2[3] = 0x80;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x8000;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x800000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x80000000;
-      break;
-  }
-}
-
-inline void append_0x80_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x80;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x8000;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x800000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x80000000;
-      break;
-
-    case 4:
-      w0[1] = 0x80;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x8000;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x800000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x80000000;
-      break;
-
-    case 8:
-      w0[2] = 0x80;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x8000;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x800000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x80000000;
-      break;
-
-    case 12:
-      w0[3] = 0x80;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x8000;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x800000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x80000000;
-      break;
-
-    case 16:
-      w1[0] = 0x80;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x8000;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x800000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x80000000;
-      break;
-
-    case 20:
-      w1[1] = 0x80;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x8000;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x800000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x80000000;
-      break;
-
-    case 24:
-      w1[2] = 0x80;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x8000;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x800000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x80000000;
-      break;
-
-    case 28:
-      w1[3] = 0x80;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x8000;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x800000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x80000000;
-      break;
-
-    case 32:
-      w2[0] = 0x80;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x8000;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x800000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x80000000;
-      break;
-
-    case 36:
-      w2[1] = 0x80;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x8000;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x800000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x80000000;
-      break;
-
-    case 40:
-      w2[2] = 0x80;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x8000;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x800000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x80000000;
-      break;
-
-    case 44:
-      w2[3] = 0x80;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x8000;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x800000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x80000000;
-      break;
-
-    case 48:
-      w3[0] = 0x80;
-      break;
-
-    case 49:
-      w3[0] = w3[0] | 0x8000;
-      break;
-
-    case 50:
-      w3[0] = w3[0] | 0x800000;
-      break;
-
-    case 51:
-      w3[0] = w3[0] | 0x80000000;
-      break;
-
-    case 52:
-      w3[1] = 0x80;
-      break;
-
-    case 53:
-      w3[1] = w3[1] | 0x8000;
-      break;
-
-    case 54:
-      w3[1] = w3[1] | 0x800000;
-      break;
-
-    case 55:
-      w3[1] = w3[1] | 0x80000000;
-      break;
-
-    case 56:
-      w3[2] = 0x80;
-      break;
-
-    case 57:
-      w3[2] = w3[2] | 0x8000;
-      break;
-
-    case 58:
-      w3[2] = w3[2] | 0x800000;
-      break;
-
-    case 59:
-      w3[2] = w3[2] | 0x80000000;
-      break;
-
-    case 60:
-      w3[3] = 0x80;
-      break;
-
-    case 61:
-      w3[3] = w3[3] | 0x8000;
-      break;
-
-    case 62:
-      w3[3] = w3[3] | 0x800000;
-      break;
-
-    case 63:
-      w3[3] = w3[3] | 0x80000000;
-      break;
-  }
-}
-
-inline void append_0x80_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x80;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x8000;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x800000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x80000000;
-      break;
-
-    case 4:
-      w0[1] = 0x80;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x8000;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x800000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x80000000;
-      break;
-
-    case 8:
-      w0[2] = 0x80;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x8000;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x800000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x80000000;
-      break;
-
-    case 12:
-      w0[3] = 0x80;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x8000;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x800000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x80000000;
-      break;
-
-    case 16:
-      w1[0] = 0x80;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x8000;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x800000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x80000000;
-      break;
-
-    case 20:
-      w1[1] = 0x80;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x8000;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x800000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x80000000;
-      break;
-
-    case 24:
-      w1[2] = 0x80;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x8000;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x800000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x80000000;
-      break;
-
-    case 28:
-      w1[3] = 0x80;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x8000;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x800000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x80000000;
-      break;
-
-    case 32:
-      w2[0] = 0x80;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x8000;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x800000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x80000000;
-      break;
-
-    case 36:
-      w2[1] = 0x80;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x8000;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x800000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x80000000;
-      break;
-
-    case 40:
-      w2[2] = 0x80;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x8000;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x800000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x80000000;
-      break;
-
-    case 44:
-      w2[3] = 0x80;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x8000;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x800000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x80000000;
-      break;
-
-    case 48:
-      w3[0] = 0x80;
-      break;
-
-    case 49:
-      w3[0] = w3[0] | 0x8000;
-      break;
-
-    case 50:
-      w3[0] = w3[0] | 0x800000;
-      break;
-
-    case 51:
-      w3[0] = w3[0] | 0x80000000;
-      break;
-
-    case 52:
-      w3[1] = 0x80;
-      break;
-
-    case 53:
-      w3[1] = w3[1] | 0x8000;
-      break;
-
-    case 54:
-      w3[1] = w3[1] | 0x800000;
-      break;
-
-    case 55:
-      w3[1] = w3[1] | 0x80000000;
-      break;
-
-    case 56:
-      w3[2] = 0x80;
-      break;
-
-    case 57:
-      w3[2] = w3[2] | 0x8000;
-      break;
-
-    case 58:
-      w3[2] = w3[2] | 0x800000;
-      break;
-
-    case 59:
-      w3[2] = w3[2] | 0x80000000;
-      break;
-
-    case 60:
-      w3[3] = 0x80;
-      break;
-
-    case 61:
-      w3[3] = w3[3] | 0x8000;
-      break;
-
-    case 62:
-      w3[3] = w3[3] | 0x800000;
-      break;
-
-    case 63:
-      w3[3] = w3[3] | 0x80000000;
-      break;
-
-    case 64:
-      w4[0] = 0x80;
-      break;
-
-    case 65:
-      w4[0] = w4[0] | 0x8000;
-      break;
-
-    case 66:
-      w4[0] = w4[0] | 0x800000;
-      break;
-
-    case 67:
-      w4[0] = w4[0] | 0x80000000;
-      break;
-
-    case 68:
-      w4[1] = 0x80;
-      break;
-
-    case 69:
-      w4[1] = w4[1] | 0x8000;
-      break;
-
-    case 70:
-      w4[1] = w4[1] | 0x800000;
-      break;
-
-    case 71:
-      w4[1] = w4[1] | 0x80000000;
-      break;
-
-    case 72:
-      w4[2] = 0x80;
-      break;
-
-    case 73:
-      w4[2] = w4[2] | 0x8000;
-      break;
-
-    case 74:
-      w4[2] = w4[2] | 0x800000;
-      break;
-
-    case 75:
-      w4[2] = w4[2] | 0x80000000;
-      break;
-
-    case 76:
-      w4[3] = 0x80;
-      break;
-
-    case 77:
-      w4[3] = w4[3] | 0x8000;
-      break;
-
-    case 78:
-      w4[3] = w4[3] | 0x800000;
-      break;
-
-    case 79:
-      w4[3] = w4[3] | 0x80000000;
-      break;
-
-    case 80:
-      w5[0] = 0x80;
-      break;
-
-    case 81:
-      w5[0] = w5[0] | 0x8000;
-      break;
-
-    case 82:
-      w5[0] = w5[0] | 0x800000;
-      break;
-
-    case 83:
-      w5[0] = w5[0] | 0x80000000;
-      break;
-
-    case 84:
-      w5[1] = 0x80;
-      break;
-
-    case 85:
-      w5[1] = w5[1] | 0x8000;
-      break;
-
-    case 86:
-      w5[1] = w5[1] | 0x800000;
-      break;
-
-    case 87:
-      w5[1] = w5[1] | 0x80000000;
-      break;
-
-    case 88:
-      w5[2] = 0x80;
-      break;
-
-    case 89:
-      w5[2] = w5[2] | 0x8000;
-      break;
-
-    case 90:
-      w5[2] = w5[2] | 0x800000;
-      break;
-
-    case 91:
-      w5[2] = w5[2] | 0x80000000;
-      break;
-
-    case 92:
-      w5[3] = 0x80;
-      break;
-
-    case 93:
-      w5[3] = w5[3] | 0x8000;
-      break;
-
-    case 94:
-      w5[3] = w5[3] | 0x800000;
-      break;
-
-    case 95:
-      w5[3] = w5[3] | 0x80000000;
-      break;
-
-    case 96:
-      w6[0] = 0x80;
-      break;
-
-    case 97:
-      w6[0] = w6[0] | 0x8000;
-      break;
-
-    case 98:
-      w6[0] = w6[0] | 0x800000;
-      break;
-
-    case 99:
-      w6[0] = w6[0] | 0x80000000;
-      break;
-
-    case 100:
-      w6[1] = 0x80;
-      break;
-
-    case 101:
-      w6[1] = w6[1] | 0x8000;
-      break;
-
-    case 102:
-      w6[1] = w6[1] | 0x800000;
-      break;
-
-    case 103:
-      w6[1] = w6[1] | 0x80000000;
-      break;
-
-    case 104:
-      w6[2] = 0x80;
-      break;
-
-    case 105:
-      w6[2] = w6[2] | 0x8000;
-      break;
-
-    case 106:
-      w6[2] = w6[2] | 0x800000;
-      break;
-
-    case 107:
-      w6[2] = w6[2] | 0x80000000;
-      break;
-
-    case 108:
-      w6[3] = 0x80;
-      break;
-
-    case 109:
-      w6[3] = w6[3] | 0x8000;
-      break;
-
-    case 110:
-      w6[3] = w6[3] | 0x800000;
-      break;
-
-    case 111:
-      w6[3] = w6[3] | 0x80000000;
-      break;
-
-    case 112:
-      w7[0] = 0x80;
-      break;
-
-    case 113:
-      w7[0] = w7[0] | 0x8000;
-      break;
-
-    case 114:
-      w7[0] = w7[0] | 0x800000;
-      break;
-
-    case 115:
-      w7[0] = w7[0] | 0x80000000;
-      break;
-
-    case 116:
-      w7[1] = 0x80;
-      break;
-
-    case 117:
-      w7[1] = w7[1] | 0x8000;
-      break;
-
-    case 118:
-      w7[1] = w7[1] | 0x800000;
-      break;
-
-    case 119:
-      w7[1] = w7[1] | 0x80000000;
-      break;
-
-    case 120:
-      w7[2] = 0x80;
-      break;
-
-    case 121:
-      w7[2] = w7[2] | 0x8000;
-      break;
-
-    case 122:
-      w7[2] = w7[2] | 0x800000;
-      break;
-
-    case 123:
-      w7[2] = w7[2] | 0x80000000;
-      break;
-
-    case 124:
-      w7[3] = 0x80;
-      break;
-
-    case 125:
-      w7[3] = w7[3] | 0x8000;
-      break;
-
-    case 126:
-      w7[3] = w7[3] | 0x800000;
-      break;
-
-    case 127:
-      w7[3] = w7[3] | 0x80000000;
-      break;
-  }
-}
-
-inline void append_0x80_1x16 (u32x w[16], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w[ 0] = 0x80;
-      break;
-
-    case 1:
-      w[ 0] = w[ 0] | 0x8000;
-      break;
-
-    case 2:
-      w[ 0] = w[ 0] | 0x800000;
-      break;
-
-    case 3:
-      w[ 0] = w[ 0] | 0x80000000;
-      break;
-
-    case 4:
-      w[ 1] = 0x80;
-      break;
-
-    case 5:
-      w[ 1] = w[ 1] | 0x8000;
-      break;
-
-    case 6:
-      w[ 1] = w[ 1] | 0x800000;
-      break;
-
-    case 7:
-      w[ 1] = w[ 1] | 0x80000000;
-      break;
-
-    case 8:
-      w[ 2] = 0x80;
-      break;
-
-    case 9:
-      w[ 2] = w[ 2] | 0x8000;
-      break;
-
-    case 10:
-      w[ 2] = w[ 2] | 0x800000;
-      break;
-
-    case 11:
-      w[ 2] = w[ 2] | 0x80000000;
-      break;
-
-    case 12:
-      w[ 3] = 0x80;
-      break;
-
-    case 13:
-      w[ 3] = w[ 3] | 0x8000;
-      break;
-
-    case 14:
-      w[ 3] = w[ 3] | 0x800000;
-      break;
-
-    case 15:
-      w[ 3] = w[ 3] | 0x80000000;
-      break;
-
-    case 16:
-      w[ 4] = 0x80;
-      break;
-
-    case 17:
-      w[ 4] = w[ 4] | 0x8000;
-      break;
-
-    case 18:
-      w[ 4] = w[ 4] | 0x800000;
-      break;
-
-    case 19:
-      w[ 4] = w[ 4] | 0x80000000;
-      break;
-
-    case 20:
-      w[ 5] = 0x80;
-      break;
-
-    case 21:
-      w[ 5] = w[ 5] | 0x8000;
-      break;
-
-    case 22:
-      w[ 5] = w[ 5] | 0x800000;
-      break;
-
-    case 23:
-      w[ 5] = w[ 5] | 0x80000000;
-      break;
-
-    case 24:
-      w[ 6] = 0x80;
-      break;
-
-    case 25:
-      w[ 6] = w[ 6] | 0x8000;
-      break;
-
-    case 26:
-      w[ 6] = w[ 6] | 0x800000;
-      break;
-
-    case 27:
-      w[ 6] = w[ 6] | 0x80000000;
-      break;
-
-    case 28:
-      w[ 7] = 0x80;
-      break;
-
-    case 29:
-      w[ 7] = w[ 7] | 0x8000;
-      break;
-
-    case 30:
-      w[ 7] = w[ 7] | 0x800000;
-      break;
-
-    case 31:
-      w[ 7] = w[ 7] | 0x80000000;
-      break;
-
-    case 32:
-      w[ 8] = 0x80;
-      break;
-
-    case 33:
-      w[ 8] = w[ 8] | 0x8000;
-      break;
-
-    case 34:
-      w[ 8] = w[ 8] | 0x800000;
-      break;
-
-    case 35:
-      w[ 8] = w[ 8] | 0x80000000;
-      break;
-
-    case 36:
-      w[ 9] = 0x80;
-      break;
-
-    case 37:
-      w[ 9] = w[ 9] | 0x8000;
-      break;
-
-    case 38:
-      w[ 9] = w[ 9] | 0x800000;
-      break;
-
-    case 39:
-      w[ 9] = w[ 9] | 0x80000000;
-      break;
-
-    case 40:
-      w[10] = 0x80;
-      break;
-
-    case 41:
-      w[10] = w[10] | 0x8000;
-      break;
-
-    case 42:
-      w[10] = w[10] | 0x800000;
-      break;
-
-    case 43:
-      w[10] = w[10] | 0x80000000;
-      break;
-
-    case 44:
-      w[11] = 0x80;
-      break;
-
-    case 45:
-      w[11] = w[11] | 0x8000;
-      break;
-
-    case 46:
-      w[11] = w[11] | 0x800000;
-      break;
-
-    case 47:
-      w[11] = w[11] | 0x80000000;
-      break;
-
-    case 48:
-      w[12] = 0x80;
-      break;
-
-    case 49:
-      w[12] = w[12] | 0x8000;
-      break;
-
-    case 50:
-      w[12] = w[12] | 0x800000;
-      break;
-
-    case 51:
-      w[12] = w[12] | 0x80000000;
-      break;
-
-    case 52:
-      w[13] = 0x80;
-      break;
-
-    case 53:
-      w[13] = w[13] | 0x8000;
-      break;
-
-    case 54:
-      w[13] = w[13] | 0x800000;
-      break;
-
-    case 55:
-      w[13] = w[13] | 0x80000000;
-      break;
-
-    case 56:
-      w[14] = 0x80;
-      break;
-
-    case 57:
-      w[14] = w[14] | 0x8000;
-      break;
-
-    case 58:
-      w[14] = w[14] | 0x800000;
-      break;
-
-    case 59:
-      w[14] = w[14] | 0x80000000;
-      break;
-
-    case 60:
-      w[15] = 0x80;
-      break;
-
-    case 61:
-      w[15] = w[15] | 0x8000;
-      break;
-
-    case 62:
-      w[15] = w[15] | 0x800000;
-      break;
-
-    case 63:
-      w[15] = w[15] | 0x80000000;
-      break;
-  }
-}
-
-inline void switch_buffer_by_offset_le (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
-{
-  #if defined IS_AMD || defined IS_GENERIC
-  const int offset_mod_4 = offset & 3;
-
-  const int offset_minus_4 = 4 - offset;
-
-  switch (offset / 4)
-  {
-    case 0:
-      w3[2] = amd_bytealign (    0, w3[1], offset_minus_4);
-      w3[1] = amd_bytealign (w3[1], w3[0], offset_minus_4);
-      w3[0] = amd_bytealign (w3[0], w2[3], offset_minus_4);
-      w2[3] = amd_bytealign (w2[3], w2[2], offset_minus_4);
-      w2[2] = amd_bytealign (w2[2], w2[1], offset_minus_4);
-      w2[1] = amd_bytealign (w2[1], w2[0], offset_minus_4);
-      w2[0] = amd_bytealign (w2[0], w1[3], offset_minus_4);
-      w1[3] = amd_bytealign (w1[3], w1[2], offset_minus_4);
-      w1[2] = amd_bytealign (w1[2], w1[1], offset_minus_4);
-      w1[1] = amd_bytealign (w1[1], w1[0], offset_minus_4);
-      w1[0] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w0[3] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w0[2] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w0[1] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w0[0] = amd_bytealign (w0[0],     0, offset_minus_4);
-
-      if (offset_mod_4 == 0)
-      {
-        w0[0] = w0[1];
-        w0[1] = w0[2];
-        w0[2] = w0[3];
-        w0[3] = w1[0];
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 1:
-      w3[2] = amd_bytealign (    0, w3[0], offset_minus_4);
-      w3[1] = amd_bytealign (w3[0], w2[3], offset_minus_4);
-      w3[0] = amd_bytealign (w2[3], w2[2], offset_minus_4);
-      w2[3] = amd_bytealign (w2[2], w2[1], offset_minus_4);
-      w2[2] = amd_bytealign (w2[1], w2[0], offset_minus_4);
-      w2[1] = amd_bytealign (w2[0], w1[3], offset_minus_4);
-      w2[0] = amd_bytealign (w1[3], w1[2], offset_minus_4);
-      w1[3] = amd_bytealign (w1[2], w1[1], offset_minus_4);
-      w1[2] = amd_bytealign (w1[1], w1[0], offset_minus_4);
-      w1[1] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w1[0] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w0[3] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w0[2] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w0[1] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w0[1] = w0[2];
-        w0[2] = w0[3];
-        w0[3] = w1[0];
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 2:
-      w3[2] = amd_bytealign (    0, w2[3], offset_minus_4);
-      w3[1] = amd_bytealign (w2[3], w2[2], offset_minus_4);
-      w3[0] = amd_bytealign (w2[2], w2[1], offset_minus_4);
-      w2[3] = amd_bytealign (w2[1], w2[0], offset_minus_4);
-      w2[2] = amd_bytealign (w2[0], w1[3], offset_minus_4);
-      w2[1] = amd_bytealign (w1[3], w1[2], offset_minus_4);
-      w2[0] = amd_bytealign (w1[2], w1[1], offset_minus_4);
-      w1[3] = amd_bytealign (w1[1], w1[0], offset_minus_4);
-      w1[2] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w1[1] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w1[0] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w0[3] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w0[2] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w0[2] = w0[3];
-        w0[3] = w1[0];
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 3:
-      w3[2] = amd_bytealign (    0, w2[2], offset_minus_4);
-      w3[1] = amd_bytealign (w2[2], w2[1], offset_minus_4);
-      w3[0] = amd_bytealign (w2[1], w2[0], offset_minus_4);
-      w2[3] = amd_bytealign (w2[0], w1[3], offset_minus_4);
-      w2[2] = amd_bytealign (w1[3], w1[2], offset_minus_4);
-      w2[1] = amd_bytealign (w1[2], w1[1], offset_minus_4);
-      w2[0] = amd_bytealign (w1[1], w1[0], offset_minus_4);
-      w1[3] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w1[2] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w1[1] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w1[0] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w0[3] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w0[3] = w1[0];
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 4:
-      w3[2] = amd_bytealign (    0, w2[1], offset_minus_4);
-      w3[1] = amd_bytealign (w2[1], w2[0], offset_minus_4);
-      w3[0] = amd_bytealign (w2[0], w1[3], offset_minus_4);
-      w2[3] = amd_bytealign (w1[3], w1[2], offset_minus_4);
-      w2[2] = amd_bytealign (w1[2], w1[1], offset_minus_4);
-      w2[1] = amd_bytealign (w1[1], w1[0], offset_minus_4);
-      w2[0] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w1[3] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w1[2] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w1[1] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w1[0] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 5:
-      w3[2] = amd_bytealign (    0, w2[0], offset_minus_4);
-      w3[1] = amd_bytealign (w2[0], w1[3], offset_minus_4);
-      w3[0] = amd_bytealign (w1[3], w1[2], offset_minus_4);
-      w2[3] = amd_bytealign (w1[2], w1[1], offset_minus_4);
-      w2[2] = amd_bytealign (w1[1], w1[0], offset_minus_4);
-      w2[1] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w2[0] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w1[3] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w1[2] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w1[1] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 6:
-      w3[2] = amd_bytealign (    0, w1[3], offset_minus_4);
-      w3[1] = amd_bytealign (w1[3], w1[2], offset_minus_4);
-      w3[0] = amd_bytealign (w1[2], w1[1], offset_minus_4);
-      w2[3] = amd_bytealign (w1[1], w1[0], offset_minus_4);
-      w2[2] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w2[1] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w2[0] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w1[3] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w1[2] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 7:
-      w3[2] = amd_bytealign (    0, w1[2], offset_minus_4);
-      w3[1] = amd_bytealign (w1[2], w1[1], offset_minus_4);
-      w3[0] = amd_bytealign (w1[1], w1[0], offset_minus_4);
-      w2[3] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w2[2] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w2[1] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w2[0] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w1[3] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 8:
-      w3[2] = amd_bytealign (    0, w1[1], offset_minus_4);
-      w3[1] = amd_bytealign (w1[1], w1[0], offset_minus_4);
-      w3[0] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w2[3] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w2[2] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w2[1] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w2[0] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 9:
-      w3[2] = amd_bytealign (    0, w1[0], offset_minus_4);
-      w3[1] = amd_bytealign (w1[0], w0[3], offset_minus_4);
-      w3[0] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w2[3] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w2[2] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w2[1] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 10:
-      w3[2] = amd_bytealign (    0, w0[3], offset_minus_4);
-      w3[1] = amd_bytealign (w0[3], w0[2], offset_minus_4);
-      w3[0] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w2[3] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w2[2] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 11:
-      w3[2] = amd_bytealign (    0, w0[2], offset_minus_4);
-      w3[1] = amd_bytealign (w0[2], w0[1], offset_minus_4);
-      w3[0] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w2[3] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 12:
-      w3[2] = amd_bytealign (    0, w0[1], offset_minus_4);
-      w3[1] = amd_bytealign (w0[1], w0[0], offset_minus_4);
-      w3[0] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 13:
-      w3[2] = amd_bytealign (    0, w0[0], offset_minus_4);
-      w3[1] = amd_bytealign (w0[0],     0, offset_minus_4);
-      w3[0] = 0;
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-  }
-  #endif
-
-  #ifdef IS_NV
-  const int offset_minus_4 = 4 - (offset % 4);
-
-  const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff;
-
-  switch (offset / 4)
-  {
-    case 0:
-      w3[1] = __byte_perm (w3[0], w3[1], selector);
-      w3[0] = __byte_perm (w2[3], w3[0], selector);
-      w2[3] = __byte_perm (w2[2], w2[3], selector);
-      w2[2] = __byte_perm (w2[1], w2[2], selector);
-      w2[1] = __byte_perm (w2[0], w2[1], selector);
-      w2[0] = __byte_perm (w1[3], w2[0], selector);
-      w1[3] = __byte_perm (w1[2], w1[3], selector);
-      w1[2] = __byte_perm (w1[1], w1[2], selector);
-      w1[1] = __byte_perm (w1[0], w1[1], selector);
-      w1[0] = __byte_perm (w0[3], w1[0], selector);
-      w0[3] = __byte_perm (w0[2], w0[3], selector);
-      w0[2] = __byte_perm (w0[1], w0[2], selector);
-      w0[1] = __byte_perm (w0[0], w0[1], selector);
-      w0[0] = __byte_perm (    0, w0[0], selector);
-
-      break;
-
-    case 1:
-      w3[1] = __byte_perm (w2[3], w3[0], selector);
-      w3[0] = __byte_perm (w2[2], w2[3], selector);
-      w2[3] = __byte_perm (w2[1], w2[2], selector);
-      w2[2] = __byte_perm (w2[0], w2[1], selector);
-      w2[1] = __byte_perm (w1[3], w2[0], selector);
-      w2[0] = __byte_perm (w1[2], w1[3], selector);
-      w1[3] = __byte_perm (w1[1], w1[2], selector);
-      w1[2] = __byte_perm (w1[0], w1[1], selector);
-      w1[1] = __byte_perm (w0[3], w1[0], selector);
-      w1[0] = __byte_perm (w0[2], w0[3], selector);
-      w0[3] = __byte_perm (w0[1], w0[2], selector);
-      w0[2] = __byte_perm (w0[0], w0[1], selector);
-      w0[1] = __byte_perm (    0, w0[0], selector);
-      w0[0] = 0;
-
-      break;
-
-    case 2:
-      w3[1] = __byte_perm (w2[2], w2[3], selector);
-      w3[0] = __byte_perm (w2[1], w2[2], selector);
-      w2[3] = __byte_perm (w2[0], w2[1], selector);
-      w2[2] = __byte_perm (w1[3], w2[0], selector);
-      w2[1] = __byte_perm (w1[2], w1[3], selector);
-      w2[0] = __byte_perm (w1[1], w1[2], selector);
-      w1[3] = __byte_perm (w1[0], w1[1], selector);
-      w1[2] = __byte_perm (w0[3], w1[0], selector);
-      w1[1] = __byte_perm (w0[2], w0[3], selector);
-      w1[0] = __byte_perm (w0[1], w0[2], selector);
-      w0[3] = __byte_perm (w0[0], w0[1], selector);
-      w0[2] = __byte_perm (    0, w0[0], selector);
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 3:
-      w3[1] = __byte_perm (w2[1], w2[2], selector);
-      w3[0] = __byte_perm (w2[0], w2[1], selector);
-      w2[3] = __byte_perm (w1[3], w2[0], selector);
-      w2[2] = __byte_perm (w1[2], w1[3], selector);
-      w2[1] = __byte_perm (w1[1], w1[2], selector);
-      w2[0] = __byte_perm (w1[0], w1[1], selector);
-      w1[3] = __byte_perm (w0[3], w1[0], selector);
-      w1[2] = __byte_perm (w0[2], w0[3], selector);
-      w1[1] = __byte_perm (w0[1], w0[2], selector);
-      w1[0] = __byte_perm (w0[0], w0[1], selector);
-      w0[3] = __byte_perm (    0, w0[0], selector);
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 4:
-      w3[1] = __byte_perm (w2[0], w2[1], selector);
-      w3[0] = __byte_perm (w1[3], w2[0], selector);
-      w2[3] = __byte_perm (w1[2], w1[3], selector);
-      w2[2] = __byte_perm (w1[1], w1[2], selector);
-      w2[1] = __byte_perm (w1[0], w1[1], selector);
-      w2[0] = __byte_perm (w0[3], w1[0], selector);
-      w1[3] = __byte_perm (w0[2], w0[3], selector);
-      w1[2] = __byte_perm (w0[1], w0[2], selector);
-      w1[1] = __byte_perm (w0[0], w0[1], selector);
-      w1[0] = __byte_perm (    0, w0[0], selector);
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 5:
-      w3[1] = __byte_perm (w1[3], w2[0], selector);
-      w3[0] = __byte_perm (w1[2], w1[3], selector);
-      w2[3] = __byte_perm (w1[1], w1[2], selector);
-      w2[2] = __byte_perm (w1[0], w1[1], selector);
-      w2[1] = __byte_perm (w0[3], w1[0], selector);
-      w2[0] = __byte_perm (w0[2], w0[3], selector);
-      w1[3] = __byte_perm (w0[1], w0[2], selector);
-      w1[2] = __byte_perm (w0[0], w0[1], selector);
-      w1[1] = __byte_perm (    0, w0[0], selector);
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 6:
-      w3[1] = __byte_perm (w1[2], w1[3], selector);
-      w3[0] = __byte_perm (w1[1], w1[2], selector);
-      w2[3] = __byte_perm (w1[0], w1[1], selector);
-      w2[2] = __byte_perm (w0[3], w1[0], selector);
-      w2[1] = __byte_perm (w0[2], w0[3], selector);
-      w2[0] = __byte_perm (w0[1], w0[2], selector);
-      w1[3] = __byte_perm (w0[0], w0[1], selector);
-      w1[2] = __byte_perm (    0, w0[0], selector);
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 7:
-      w3[1] = __byte_perm (w1[1], w1[2], selector);
-      w3[0] = __byte_perm (w1[0], w1[1], selector);
-      w2[3] = __byte_perm (w0[3], w1[0], selector);
-      w2[2] = __byte_perm (w0[2], w0[3], selector);
-      w2[1] = __byte_perm (w0[1], w0[2], selector);
-      w2[0] = __byte_perm (w0[0], w0[1], selector);
-      w1[3] = __byte_perm (    0, w0[0], selector);
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 8:
-      w3[1] = __byte_perm (w1[0], w1[1], selector);
-      w3[0] = __byte_perm (w0[3], w1[0], selector);
-      w2[3] = __byte_perm (w0[2], w0[3], selector);
-      w2[2] = __byte_perm (w0[1], w0[2], selector);
-      w2[1] = __byte_perm (w0[0], w0[1], selector);
-      w2[0] = __byte_perm (    0, w0[0], selector);
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 9:
-      w3[1] = __byte_perm (w0[3], w1[0], selector);
-      w3[0] = __byte_perm (w0[2], w0[3], selector);
-      w2[3] = __byte_perm (w0[1], w0[2], selector);
-      w2[2] = __byte_perm (w0[0], w0[1], selector);
-      w2[1] = __byte_perm (    0, w0[0], selector);
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 10:
-      w3[1] = __byte_perm (w0[2], w0[3], selector);
-      w3[0] = __byte_perm (w0[1], w0[2], selector);
-      w2[3] = __byte_perm (w0[0], w0[1], selector);
-      w2[2] = __byte_perm (    0, w0[0], selector);
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 11:
-      w3[1] = __byte_perm (w0[1], w0[2], selector);
-      w3[0] = __byte_perm (w0[0], w0[1], selector);
-      w2[3] = __byte_perm (    0, w0[0], selector);
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 12:
-      w3[1] = __byte_perm (w0[0], w0[1], selector);
-      w3[0] = __byte_perm (    0, w0[0], selector);
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 13:
-      w3[1] = __byte_perm (    0, w0[0], selector);
-      w3[0] = 0;
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-  }
-  #endif
-}
-
-inline void switch_buffer_by_offset_be (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
-{
-  #if defined IS_AMD || defined IS_GENERIC
-  switch (offset / 4)
-  {
-    case 0:
-      w3[2] = amd_bytealign (w3[1],     0, offset);
-      w3[1] = amd_bytealign (w3[0], w3[1], offset);
-      w3[0] = amd_bytealign (w2[3], w3[0], offset);
-      w2[3] = amd_bytealign (w2[2], w2[3], offset);
-      w2[2] = amd_bytealign (w2[1], w2[2], offset);
-      w2[1] = amd_bytealign (w2[0], w2[1], offset);
-      w2[0] = amd_bytealign (w1[3], w2[0], offset);
-      w1[3] = amd_bytealign (w1[2], w1[3], offset);
-      w1[2] = amd_bytealign (w1[1], w1[2], offset);
-      w1[1] = amd_bytealign (w1[0], w1[1], offset);
-      w1[0] = amd_bytealign (w0[3], w1[0], offset);
-      w0[3] = amd_bytealign (w0[2], w0[3], offset);
-      w0[2] = amd_bytealign (w0[1], w0[2], offset);
-      w0[1] = amd_bytealign (w0[0], w0[1], offset);
-      w0[0] = amd_bytealign (    0, w0[0], offset);
-      break;
-
-    case 1:
-      w3[2] = amd_bytealign (w3[0],     0, offset);
-      w3[1] = amd_bytealign (w2[3], w3[0], offset);
-      w3[0] = amd_bytealign (w2[2], w2[3], offset);
-      w2[3] = amd_bytealign (w2[1], w2[2], offset);
-      w2[2] = amd_bytealign (w2[0], w2[1], offset);
-      w2[1] = amd_bytealign (w1[3], w2[0], offset);
-      w2[0] = amd_bytealign (w1[2], w1[3], offset);
-      w1[3] = amd_bytealign (w1[1], w1[2], offset);
-      w1[2] = amd_bytealign (w1[0], w1[1], offset);
-      w1[1] = amd_bytealign (w0[3], w1[0], offset);
-      w1[0] = amd_bytealign (w0[2], w0[3], offset);
-      w0[3] = amd_bytealign (w0[1], w0[2], offset);
-      w0[2] = amd_bytealign (w0[0], w0[1], offset);
-      w0[1] = amd_bytealign (    0, w0[0], offset);
-      w0[0] = 0;
-      break;
-
-    case 2:
-      w3[2] = amd_bytealign (w2[3],     0, offset);
-      w3[1] = amd_bytealign (w2[2], w2[3], offset);
-      w3[0] = amd_bytealign (w2[1], w2[2], offset);
-      w2[3] = amd_bytealign (w2[0], w2[1], offset);
-      w2[2] = amd_bytealign (w1[3], w2[0], offset);
-      w2[1] = amd_bytealign (w1[2], w1[3], offset);
-      w2[0] = amd_bytealign (w1[1], w1[2], offset);
-      w1[3] = amd_bytealign (w1[0], w1[1], offset);
-      w1[2] = amd_bytealign (w0[3], w1[0], offset);
-      w1[1] = amd_bytealign (w0[2], w0[3], offset);
-      w1[0] = amd_bytealign (w0[1], w0[2], offset);
-      w0[3] = amd_bytealign (w0[0], w0[1], offset);
-      w0[2] = amd_bytealign (    0, w0[0], offset);
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 3:
-      w3[2] = amd_bytealign (w2[2],     0, offset);
-      w3[1] = amd_bytealign (w2[1], w2[2], offset);
-      w3[0] = amd_bytealign (w2[0], w2[1], offset);
-      w2[3] = amd_bytealign (w1[3], w2[0], offset);
-      w2[2] = amd_bytealign (w1[2], w1[3], offset);
-      w2[1] = amd_bytealign (w1[1], w1[2], offset);
-      w2[0] = amd_bytealign (w1[0], w1[1], offset);
-      w1[3] = amd_bytealign (w0[3], w1[0], offset);
-      w1[2] = amd_bytealign (w0[2], w0[3], offset);
-      w1[1] = amd_bytealign (w0[1], w0[2], offset);
-      w1[0] = amd_bytealign (w0[0], w0[1], offset);
-      w0[3] = amd_bytealign (    0, w0[0], offset);
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 4:
-      w3[2] = amd_bytealign (w2[1],     0, offset);
-      w3[1] = amd_bytealign (w2[0], w2[1], offset);
-      w3[0] = amd_bytealign (w1[3], w2[0], offset);
-      w2[3] = amd_bytealign (w1[2], w1[3], offset);
-      w2[2] = amd_bytealign (w1[1], w1[2], offset);
-      w2[1] = amd_bytealign (w1[0], w1[1], offset);
-      w2[0] = amd_bytealign (w0[3], w1[0], offset);
-      w1[3] = amd_bytealign (w0[2], w0[3], offset);
-      w1[2] = amd_bytealign (w0[1], w0[2], offset);
-      w1[1] = amd_bytealign (w0[0], w0[1], offset);
-      w1[0] = amd_bytealign (    0, w0[0], offset);
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 5:
-      w3[2] = amd_bytealign (w2[0],     0, offset);
-      w3[1] = amd_bytealign (w1[3], w2[0], offset);
-      w3[0] = amd_bytealign (w1[2], w1[3], offset);
-      w2[3] = amd_bytealign (w1[1], w1[2], offset);
-      w2[2] = amd_bytealign (w1[0], w1[1], offset);
-      w2[1] = amd_bytealign (w0[3], w1[0], offset);
-      w2[0] = amd_bytealign (w0[2], w0[3], offset);
-      w1[3] = amd_bytealign (w0[1], w0[2], offset);
-      w1[2] = amd_bytealign (w0[0], w0[1], offset);
-      w1[1] = amd_bytealign (    0, w0[0], offset);
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 6:
-      w3[2] = amd_bytealign (w1[3],     0, offset);
-      w3[1] = amd_bytealign (w1[2], w1[3], offset);
-      w3[0] = amd_bytealign (w1[1], w1[2], offset);
-      w2[3] = amd_bytealign (w1[0], w1[1], offset);
-      w2[2] = amd_bytealign (w0[3], w1[0], offset);
-      w2[1] = amd_bytealign (w0[2], w0[3], offset);
-      w2[0] = amd_bytealign (w0[1], w0[2], offset);
-      w1[3] = amd_bytealign (w0[0], w0[1], offset);
-      w1[2] = amd_bytealign (    0, w0[0], offset);
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 7:
-      w3[2] = amd_bytealign (w1[2],     0, offset);
-      w3[1] = amd_bytealign (w1[1], w1[2], offset);
-      w3[0] = amd_bytealign (w1[0], w1[1], offset);
-      w2[3] = amd_bytealign (w0[3], w1[0], offset);
-      w2[2] = amd_bytealign (w0[2], w0[3], offset);
-      w2[1] = amd_bytealign (w0[1], w0[2], offset);
-      w2[0] = amd_bytealign (w0[0], w0[1], offset);
-      w1[3] = amd_bytealign (    0, w0[0], offset);
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 8:
-      w3[2] = amd_bytealign (w1[1],     0, offset);
-      w3[1] = amd_bytealign (w1[0], w1[1], offset);
-      w3[0] = amd_bytealign (w0[3], w1[0], offset);
-      w2[3] = amd_bytealign (w0[2], w0[3], offset);
-      w2[2] = amd_bytealign (w0[1], w0[2], offset);
-      w2[1] = amd_bytealign (w0[0], w0[1], offset);
-      w2[0] = amd_bytealign (    0, w0[0], offset);
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 9:
-      w3[2] = amd_bytealign (w1[0],     0, offset);
-      w3[1] = amd_bytealign (w0[3], w1[0], offset);
-      w3[0] = amd_bytealign (w0[2], w0[3], offset);
-      w2[3] = amd_bytealign (w0[1], w0[2], offset);
-      w2[2] = amd_bytealign (w0[0], w0[1], offset);
-      w2[1] = amd_bytealign (    0, w0[0], offset);
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 10:
-      w3[2] = amd_bytealign (w0[3],     0, offset);
-      w3[1] = amd_bytealign (w0[2], w0[3], offset);
-      w3[0] = amd_bytealign (w0[1], w0[2], offset);
-      w2[3] = amd_bytealign (w0[0], w0[1], offset);
-      w2[2] = amd_bytealign (    0, w0[0], offset);
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 11:
-      w3[2] = amd_bytealign (w0[2],     0, offset);
-      w3[1] = amd_bytealign (w0[1], w0[2], offset);
-      w3[0] = amd_bytealign (w0[0], w0[1], offset);
-      w2[3] = amd_bytealign (    0, w0[0], offset);
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 12:
-      w3[2] = amd_bytealign (w0[1],     0, offset);
-      w3[1] = amd_bytealign (w0[0], w0[1], offset);
-      w3[0] = amd_bytealign (    0, w0[0], offset);
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 13:
-      w3[2] = amd_bytealign (w0[0],     0, offset);
-      w3[1] = amd_bytealign (    0, w0[0], offset);
-      w3[0] = 0;
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-  }
-  #endif
-
-  #ifdef IS_NV
-  const int selector = (0x76543210 >> ((offset & 3) * 4)) & 0xffff;
-
-  switch (offset / 4)
-  {
-    case 0:
-      w3[1] = __byte_perm (w3[1], w3[0], selector);
-      w3[0] = __byte_perm (w3[0], w2[3], selector);
-      w2[3] = __byte_perm (w2[3], w2[2], selector);
-      w2[2] = __byte_perm (w2[2], w2[1], selector);
-      w2[1] = __byte_perm (w2[1], w2[0], selector);
-      w2[0] = __byte_perm (w2[0], w1[3], selector);
-      w1[3] = __byte_perm (w1[3], w1[2], selector);
-      w1[2] = __byte_perm (w1[2], w1[1], selector);
-      w1[1] = __byte_perm (w1[1], w1[0], selector);
-      w1[0] = __byte_perm (w1[0], w0[3], selector);
-      w0[3] = __byte_perm (w0[3], w0[2], selector);
-      w0[2] = __byte_perm (w0[2], w0[1], selector);
-      w0[1] = __byte_perm (w0[1], w0[0], selector);
-      w0[0] = __byte_perm (w0[0],     0, selector);
-      break;
-
-    case 1:
-      w3[1] = __byte_perm (w3[0], w2[3], selector);
-      w3[0] = __byte_perm (w2[3], w2[2], selector);
-      w2[3] = __byte_perm (w2[2], w2[1], selector);
-      w2[2] = __byte_perm (w2[1], w2[0], selector);
-      w2[1] = __byte_perm (w2[0], w1[3], selector);
-      w2[0] = __byte_perm (w1[3], w1[2], selector);
-      w1[3] = __byte_perm (w1[2], w1[1], selector);
-      w1[2] = __byte_perm (w1[1], w1[0], selector);
-      w1[1] = __byte_perm (w1[0], w0[3], selector);
-      w1[0] = __byte_perm (w0[3], w0[2], selector);
-      w0[3] = __byte_perm (w0[2], w0[1], selector);
-      w0[2] = __byte_perm (w0[1], w0[0], selector);
-      w0[1] = __byte_perm (w0[0],     0, selector);
-      w0[0] = 0;
-      break;
-
-    case 2:
-      w3[1] = __byte_perm (w2[3], w2[2], selector);
-      w3[0] = __byte_perm (w2[2], w2[1], selector);
-      w2[3] = __byte_perm (w2[1], w2[0], selector);
-      w2[2] = __byte_perm (w2[0], w1[3], selector);
-      w2[1] = __byte_perm (w1[3], w1[2], selector);
-      w2[0] = __byte_perm (w1[2], w1[1], selector);
-      w1[3] = __byte_perm (w1[1], w1[0], selector);
-      w1[2] = __byte_perm (w1[0], w0[3], selector);
-      w1[1] = __byte_perm (w0[3], w0[2], selector);
-      w1[0] = __byte_perm (w0[2], w0[1], selector);
-      w0[3] = __byte_perm (w0[1], w0[0], selector);
-      w0[2] = __byte_perm (w0[0],     0, selector);
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 3:
-      w3[1] = __byte_perm (w2[2], w2[1], selector);
-      w3[0] = __byte_perm (w2[1], w2[0], selector);
-      w2[3] = __byte_perm (w2[0], w1[3], selector);
-      w2[2] = __byte_perm (w1[3], w1[2], selector);
-      w2[1] = __byte_perm (w1[2], w1[1], selector);
-      w2[0] = __byte_perm (w1[1], w1[0], selector);
-      w1[3] = __byte_perm (w1[0], w0[3], selector);
-      w1[2] = __byte_perm (w0[3], w0[2], selector);
-      w1[1] = __byte_perm (w0[2], w0[1], selector);
-      w1[0] = __byte_perm (w0[1], w0[0], selector);
-      w0[3] = __byte_perm (w0[0],     0, selector);
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 4:
-      w3[1] = __byte_perm (w2[1], w2[0], selector);
-      w3[0] = __byte_perm (w2[0], w1[3], selector);
-      w2[3] = __byte_perm (w1[3], w1[2], selector);
-      w2[2] = __byte_perm (w1[2], w1[1], selector);
-      w2[1] = __byte_perm (w1[1], w1[0], selector);
-      w2[0] = __byte_perm (w1[0], w0[3], selector);
-      w1[3] = __byte_perm (w0[3], w0[2], selector);
-      w1[2] = __byte_perm (w0[2], w0[1], selector);
-      w1[1] = __byte_perm (w0[1], w0[0], selector);
-      w1[0] = __byte_perm (w0[0],     0, selector);
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 5:
-      w3[1] = __byte_perm (w2[0], w1[3], selector);
-      w3[0] = __byte_perm (w1[3], w1[2], selector);
-      w2[3] = __byte_perm (w1[2], w1[1], selector);
-      w2[2] = __byte_perm (w1[1], w1[0], selector);
-      w2[1] = __byte_perm (w1[0], w0[3], selector);
-      w2[0] = __byte_perm (w0[3], w0[2], selector);
-      w1[3] = __byte_perm (w0[2], w0[1], selector);
-      w1[2] = __byte_perm (w0[1], w0[0], selector);
-      w1[1] = __byte_perm (w0[0],     0, selector);
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 6:
-      w3[1] = __byte_perm (w1[3], w1[2], selector);
-      w3[0] = __byte_perm (w1[2], w1[1], selector);
-      w2[3] = __byte_perm (w1[1], w1[0], selector);
-      w2[2] = __byte_perm (w1[0], w0[3], selector);
-      w2[1] = __byte_perm (w0[3], w0[2], selector);
-      w2[0] = __byte_perm (w0[2], w0[1], selector);
-      w1[3] = __byte_perm (w0[1], w0[0], selector);
-      w1[2] = __byte_perm (w0[0],     0, selector);
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 7:
-      w3[1] = __byte_perm (w1[2], w1[1], selector);
-      w3[0] = __byte_perm (w1[1], w1[0], selector);
-      w2[3] = __byte_perm (w1[0], w0[3], selector);
-      w2[2] = __byte_perm (w0[3], w0[2], selector);
-      w2[1] = __byte_perm (w0[2], w0[1], selector);
-      w2[0] = __byte_perm (w0[1], w0[0], selector);
-      w1[3] = __byte_perm (w0[0],     0, selector);
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 8:
-      w3[1] = __byte_perm (w1[1], w1[0], selector);
-      w3[0] = __byte_perm (w1[0], w0[3], selector);
-      w2[3] = __byte_perm (w0[3], w0[2], selector);
-      w2[2] = __byte_perm (w0[2], w0[1], selector);
-      w2[1] = __byte_perm (w0[1], w0[0], selector);
-      w2[0] = __byte_perm (w0[0],     0, selector);
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 9:
-      w3[1] = __byte_perm (w1[0], w0[3], selector);
-      w3[0] = __byte_perm (w0[3], w0[2], selector);
-      w2[3] = __byte_perm (w0[2], w0[1], selector);
-      w2[2] = __byte_perm (w0[1], w0[0], selector);
-      w2[1] = __byte_perm (w0[0],     0, selector);
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 10:
-      w3[1] = __byte_perm (w0[3], w0[2], selector);
-      w3[0] = __byte_perm (w0[2], w0[1], selector);
-      w2[3] = __byte_perm (w0[1], w0[0], selector);
-      w2[2] = __byte_perm (w0[0],     0, selector);
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 11:
-      w3[1] = __byte_perm (w0[2], w0[1], selector);
-      w3[0] = __byte_perm (w0[1], w0[0], selector);
-      w2[3] = __byte_perm (w0[0],     0, selector);
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 12:
-      w3[1] = __byte_perm (w0[1], w0[0], selector);
-      w3[0] = __byte_perm (w0[0],     0, selector);
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 13:
-      w3[1] = __byte_perm (w0[0],     0, selector);
-      w3[0] = 0;
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-  }
-  #endif
-}
-
-inline void overwrite_at_le (u32x sw[16], const u32x w0, const u32 salt_len)
-{
-  #if defined cl_amd_media_ops
-  switch (salt_len)
-  {
-    case  0:  sw[0] = w0;
-              break;
-    case  1:  sw[0] = amd_bytealign (w0, sw[0] << 24, 3);
-              sw[1] = amd_bytealign (sw[1] >>  8, w0, 3);
-              break;
-    case  2:  sw[0] = amd_bytealign (w0, sw[0] << 16, 2);
-              sw[1] = amd_bytealign (sw[1] >> 16, w0, 2);
-              break;
-    case  3:  sw[0] = amd_bytealign (w0, sw[0] <<  8, 1);
-              sw[1] = amd_bytealign (sw[1] >> 24, w0, 1);
-              break;
-    case  4:  sw[1] = w0;
-              break;
-    case  5:  sw[1] = amd_bytealign (w0, sw[1] << 24, 3);
-              sw[2] = amd_bytealign (sw[2] >>  8, w0, 3);
-              break;
-    case  6:  sw[1] = amd_bytealign (w0, sw[1] << 16, 2);
-              sw[2] = amd_bytealign (sw[2] >> 16, w0, 2);
-              break;
-    case  7:  sw[1] = amd_bytealign (w0, sw[1] <<  8, 1);
-              sw[2] = amd_bytealign (sw[2] >> 24, w0, 1);
-              break;
-    case  8:  sw[2] = w0;
-              break;
-    case  9:  sw[2] = amd_bytealign (w0, sw[2] << 24, 3);
-              sw[3] = amd_bytealign (sw[3] >>  8, w0, 3);
-              break;
-    case 10:  sw[2] = amd_bytealign (w0, sw[2] << 16, 2);
-              sw[3] = amd_bytealign (sw[3] >> 16, w0, 2);
-              break;
-    case 11:  sw[2] = amd_bytealign (w0, sw[2] <<  8, 1);
-              sw[3] = amd_bytealign (sw[3] >> 24, w0, 1);
-              break;
-    case 12:  sw[3] = w0;
-              break;
-    case 13:  sw[3] = amd_bytealign (w0, sw[3] << 24, 3);
-              sw[4] = amd_bytealign (sw[4] >>  8, w0, 3);
-              break;
-    case 14:  sw[3] = amd_bytealign (w0, sw[3] << 16, 2);
-              sw[4] = amd_bytealign (sw[4] >> 16, w0, 2);
-              break;
-    case 15:  sw[3] = amd_bytealign (w0, sw[3] <<  8, 1);
-              sw[4] = amd_bytealign (sw[4] >> 24, w0, 1);
-              break;
-    case 16:  sw[4] = w0;
-              break;
-    case 17:  sw[4] = amd_bytealign (w0, sw[4] << 24, 3);
-              sw[5] = amd_bytealign (sw[5] >>  8, w0, 3);
-              break;
-    case 18:  sw[4] = amd_bytealign (w0, sw[4] << 16, 2);
-              sw[5] = amd_bytealign (sw[5] >> 16, w0, 2);
-              break;
-    case 19:  sw[4] = amd_bytealign (w0, sw[4] <<  8, 1);
-              sw[5] = amd_bytealign (sw[5] >> 24, w0, 1);
-              break;
-    case 20:  sw[5] = w0;
-              break;
-    case 21:  sw[5] = amd_bytealign (w0, sw[5] << 24, 3);
-              sw[6] = amd_bytealign (sw[6] >>  8, w0, 3);
-              break;
-    case 22:  sw[5] = amd_bytealign (w0, sw[5] << 16, 2);
-              sw[6] = amd_bytealign (sw[6] >> 16, w0, 2);
-              break;
-    case 23:  sw[5] = amd_bytealign (w0, sw[5] <<  8, 1);
-              sw[6] = amd_bytealign (sw[6] >> 24, w0, 1);
-              break;
-    case 24:  sw[6] = w0;
-              break;
-    case 25:  sw[6] = amd_bytealign (w0, sw[6] << 24, 3);
-              sw[7] = amd_bytealign (sw[7] >>  8, w0, 3);
-              break;
-    case 26:  sw[6] = amd_bytealign (w0, sw[6] << 16, 2);
-              sw[7] = amd_bytealign (sw[7] >> 16, w0, 2);
-              break;
-    case 27:  sw[6] = amd_bytealign (w0, sw[6] <<  8, 1);
-              sw[7] = amd_bytealign (sw[7] >> 24, w0, 1);
-              break;
-    case 28:  sw[7] = w0;
-              break;
-    case 29:  sw[7] = amd_bytealign (w0, sw[7] << 24, 3);
-              sw[8] = amd_bytealign (sw[8] >>  8, w0, 3);
-              break;
-    case 30:  sw[7] = amd_bytealign (w0, sw[7] << 16, 2);
-              sw[8] = amd_bytealign (sw[8] >> 16, w0, 2);
-              break;
-    case 31:  sw[7] = amd_bytealign (w0, sw[7] <<  8, 1);
-              sw[8] = amd_bytealign (sw[8] >> 24, w0, 1);
-              break;
-  }
-  #else
-  switch (salt_len)
-  {
-    case  0:  sw[0] =  w0;
-              break;
-    case  1:  sw[0] = (sw[0] & 0x000000ff) | (w0 <<  8);
-              sw[1] = (sw[1] & 0xffffff00) | (w0 >> 24);
-              break;
-    case  2:  sw[0] = (sw[0] & 0x0000ffff) | (w0 << 16);
-              sw[1] = (sw[1] & 0xffff0000) | (w0 >> 16);
-              break;
-    case  3:  sw[0] = (sw[0] & 0x00ffffff) | (w0 << 24);
-              sw[1] = (sw[1] & 0xff000000) | (w0 >>  8);
-              break;
-    case  4:  sw[1] =  w0;
-              break;
-    case  5:  sw[1] = (sw[1] & 0x000000ff) | (w0 <<  8);
-              sw[2] = (sw[2] & 0xffffff00) | (w0 >> 24);
-              break;
-    case  6:  sw[1] = (sw[1] & 0x0000ffff) | (w0 << 16);
-              sw[2] = (sw[2] & 0xffff0000) | (w0 >> 16);
-              break;
-    case  7:  sw[1] = (sw[1] & 0x00ffffff) | (w0 << 24);
-              sw[2] = (sw[2] & 0xff000000) | (w0 >>  8);
-              break;
-    case  8:  sw[2] =  w0;
-              break;
-    case  9:  sw[2] = (sw[2] & 0x000000ff) | (w0 <<  8);
-              sw[3] = (sw[3] & 0xffffff00) | (w0 >> 24);
-              break;
-    case 10:  sw[2] = (sw[2] & 0x0000ffff) | (w0 << 16);
-              sw[3] = (sw[3] & 0xffff0000) | (w0 >> 16);
-              break;
-    case 11:  sw[2] = (sw[2] & 0x00ffffff) | (w0 << 24);
-              sw[3] = (sw[3] & 0xff000000) | (w0 >>  8);
-              break;
-    case 12:  sw[3] =  w0;
-              break;
-    case 13:  sw[3] = (sw[3] & 0x000000ff) | (w0 <<  8);
-              sw[4] = (sw[4] & 0xffffff00) | (w0 >> 24);
-              break;
-    case 14:  sw[3] = (sw[3] & 0x0000ffff) | (w0 << 16);
-              sw[4] = (sw[4] & 0xffff0000) | (w0 >> 16);
-              break;
-    case 15:  sw[3] = (sw[3] & 0x00ffffff) | (w0 << 24);
-              sw[4] = (sw[4] & 0xff000000) | (w0 >>  8);
-              break;
-    case 16:  sw[4] =  w0;
-              break;
-    case 17:  sw[4] = (sw[4] & 0x000000ff) | (w0 <<  8);
-              sw[5] = (sw[5] & 0xffffff00) | (w0 >> 24);
-              break;
-    case 18:  sw[4] = (sw[4] & 0x0000ffff) | (w0 << 16);
-              sw[5] = (sw[5] & 0xffff0000) | (w0 >> 16);
-              break;
-    case 19:  sw[4] = (sw[4] & 0x00ffffff) | (w0 << 24);
-              sw[5] = (sw[5] & 0xff000000) | (w0 >>  8);
-              break;
-    case 20:  sw[5] =  w0;
-              break;
-    case 21:  sw[5] = (sw[5] & 0x000000ff) | (w0 <<  8);
-              sw[6] = (sw[6] & 0xffffff00) | (w0 >> 24);
-              break;
-    case 22:  sw[5] = (sw[5] & 0x0000ffff) | (w0 << 16);
-              sw[6] = (sw[6] & 0xffff0000) | (w0 >> 16);
-              break;
-    case 23:  sw[5] = (sw[5] & 0x00ffffff) | (w0 << 24);
-              sw[6] = (sw[6] & 0xff000000) | (w0 >>  8);
-              break;
-    case 24:  sw[6] =  w0;
-              break;
-    case 25:  sw[6] = (sw[6] & 0x000000ff) | (w0 <<  8);
-              sw[7] = (sw[7] & 0xffffff00) | (w0 >> 24);
-              break;
-    case 26:  sw[6] = (sw[6] & 0x0000ffff) | (w0 << 16);
-              sw[7] = (sw[7] & 0xffff0000) | (w0 >> 16);
-              break;
-    case 27:  sw[6] = (sw[6] & 0x00ffffff) | (w0 << 24);
-              sw[7] = (sw[7] & 0xff000000) | (w0 >>  8);
-              break;
-    case 28:  sw[7] =  w0;
-              break;
-    case 29:  sw[7] = (sw[7] & 0x000000ff) | (w0 <<  8);
-              sw[8] = (sw[8] & 0xffffff00) | (w0 >> 24);
-              break;
-    case 30:  sw[7] = (sw[7] & 0x0000ffff) | (w0 << 16);
-              sw[8] = (sw[8] & 0xffff0000) | (w0 >> 16);
-              break;
-    case 31:  sw[7] = (sw[7] & 0x00ffffff) | (w0 << 24);
-              sw[8] = (sw[8] & 0xff000000) | (w0 >>  8);
-              break;
-  }
-  #endif
-}
-
-inline void overwrite_at_be (u32x sw[16], const u32x w0, const u32 salt_len)
-{
-  // would be nice to have optimization based on amd_bytealign as with _le counterpart
-
-  switch (salt_len)
-  {
-    case  0:  sw[0] =  w0;
-              break;
-    case  1:  sw[0] = (sw[0] & 0xff000000) | (w0 >>  8);
-              sw[1] = (sw[1] & 0x00ffffff) | (w0 << 24);
-              break;
-    case  2:  sw[0] = (sw[0] & 0xffff0000) | (w0 >> 16);
-              sw[1] = (sw[1] & 0x0000ffff) | (w0 << 16);
-              break;
-    case  3:  sw[0] = (sw[0] & 0xffffff00) | (w0 >> 24);
-              sw[1] = (sw[1] & 0x000000ff) | (w0 <<  8);
-              break;
-    case  4:  sw[1] =  w0;
-              break;
-    case  5:  sw[1] = (sw[1] & 0xff000000) | (w0 >>  8);
-              sw[2] = (sw[2] & 0x00ffffff) | (w0 << 24);
-              break;
-    case  6:  sw[1] = (sw[1] & 0xffff0000) | (w0 >> 16);
-              sw[2] = (sw[2] & 0x0000ffff) | (w0 << 16);
-              break;
-    case  7:  sw[1] = (sw[1] & 0xffffff00) | (w0 >> 24);
-              sw[2] = (sw[2] & 0x000000ff) | (w0 <<  8);
-              break;
-    case  8:  sw[2] =  w0;
-              break;
-    case  9:  sw[2] = (sw[2] & 0xff000000) | (w0 >>  8);
-              sw[3] = (sw[3] & 0x00ffffff) | (w0 << 24);
-              break;
-    case 10:  sw[2] = (sw[2] & 0xffff0000) | (w0 >> 16);
-              sw[3] = (sw[3] & 0x0000ffff) | (w0 << 16);
-              break;
-    case 11:  sw[2] = (sw[2] & 0xffffff00) | (w0 >> 24);
-              sw[3] = (sw[3] & 0x000000ff) | (w0 <<  8);
-              break;
-    case 12:  sw[3] =  w0;
-              break;
-    case 13:  sw[3] = (sw[3] & 0xff000000) | (w0 >>  8);
-              sw[4] = (sw[4] & 0x00ffffff) | (w0 << 24);
-              break;
-    case 14:  sw[3] = (sw[3] & 0xffff0000) | (w0 >> 16);
-              sw[4] = (sw[4] & 0x0000ffff) | (w0 << 16);
-              break;
-    case 15:  sw[3] = (sw[3] & 0xffffff00) | (w0 >> 24);
-              sw[4] = (sw[4] & 0x000000ff) | (w0 <<  8);
-              break;
-    case 16:  sw[4] =  w0;
-              break;
-    case 17:  sw[4] = (sw[4] & 0xff000000) | (w0 >>  8);
-              sw[5] = (sw[5] & 0x00ffffff) | (w0 << 24);
-              break;
-    case 18:  sw[4] = (sw[4] & 0xffff0000) | (w0 >> 16);
-              sw[5] = (sw[5] & 0x0000ffff) | (w0 << 16);
-              break;
-    case 19:  sw[4] = (sw[4] & 0xffffff00) | (w0 >> 24);
-              sw[5] = (sw[5] & 0x000000ff) | (w0 <<  8);
-              break;
-    case 20:  sw[5] =  w0;
-              break;
-    case 21:  sw[5] = (sw[5] & 0xff000000) | (w0 >>  8);
-              sw[6] = (sw[6] & 0x00ffffff) | (w0 << 24);
-              break;
-    case 22:  sw[5] = (sw[5] & 0xffff0000) | (w0 >> 16);
-              sw[6] = (sw[6] & 0x0000ffff) | (w0 << 16);
-              break;
-    case 23:  sw[5] = (sw[5] & 0xffffff00) | (w0 >> 24);
-              sw[6] = (sw[6] & 0x000000ff) | (w0 <<  8);
-              break;
-    case 24:  sw[6] =  w0;
-              break;
-    case 25:  sw[6] = (sw[6] & 0xff000000) | (w0 >>  8);
-              sw[7] = (sw[7] & 0x00ffffff) | (w0 << 24);
-              break;
-    case 26:  sw[6] = (sw[6] & 0xffff0000) | (w0 >> 16);
-              sw[7] = (sw[7] & 0x0000ffff) | (w0 << 16);
-              break;
-    case 27:  sw[6] = (sw[6] & 0xffffff00) | (w0 >> 24);
-              sw[7] = (sw[7] & 0x000000ff) | (w0 <<  8);
-              break;
-    case 28:  sw[7] =  w0;
-              break;
-    case 29:  sw[7] = (sw[7] & 0xff000000) | (w0 >>  8);
-              sw[8] = (sw[8] & 0x00ffffff) | (w0 << 24);
-              break;
-    case 30:  sw[7] = (sw[7] & 0xffff0000) | (w0 >> 16);
-              sw[8] = (sw[8] & 0x0000ffff) | (w0 << 16);
-              break;
-    case 31:  sw[7] = (sw[7] & 0xffffff00) | (w0 >> 24);
-              sw[8] = (sw[8] & 0x000000ff) | (w0 <<  8);
-              break;
-  }
-}
-
-inline void overwrite_at_le_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x wx, const u32 salt_len)
-{
-  #if defined cl_amd_media_ops
-  switch (salt_len)
-  {
-    case  0:  w0[0] = wx;
-              break;
-    case  1:  w0[0] = amd_bytealign (wx, w0[0] << 24, 3);
-              w0[1] = amd_bytealign (w0[1] >>  8, wx, 3);
-              break;
-    case  2:  w0[0] = amd_bytealign (wx, w0[0] << 16, 2);
-              w0[1] = amd_bytealign (w0[1] >> 16, wx, 2);
-              break;
-    case  3:  w0[0] = amd_bytealign (wx, w0[0] <<  8, 1);
-              w0[1] = amd_bytealign (w0[1] >> 24, wx, 1);
-              break;
-    case  4:  w0[1] = wx;
-              break;
-    case  5:  w0[1] = amd_bytealign (wx, w0[1] << 24, 3);
-              w0[2] = amd_bytealign (w0[2] >>  8, wx, 3);
-              break;
-    case  6:  w0[1] = amd_bytealign (wx, w0[1] << 16, 2);
-              w0[2] = amd_bytealign (w0[2] >> 16, wx, 2);
-              break;
-    case  7:  w0[1] = amd_bytealign (wx, w0[1] <<  8, 1);
-              w0[2] = amd_bytealign (w0[2] >> 24, wx, 1);
-              break;
-    case  8:  w0[2] = wx;
-              break;
-    case  9:  w0[2] = amd_bytealign (wx, w0[2] << 24, 3);
-              w0[3] = amd_bytealign (w0[3] >>  8, wx, 3);
-              break;
-    case 10:  w0[2] = amd_bytealign (wx, w0[2] << 16, 2);
-              w0[3] = amd_bytealign (w0[3] >> 16, wx, 2);
-              break;
-    case 11:  w0[2] = amd_bytealign (wx, w0[2] <<  8, 1);
-              w0[3] = amd_bytealign (w0[3] >> 24, wx, 1);
-              break;
-    case 12:  w0[3] = wx;
-              break;
-    case 13:  w0[3] = amd_bytealign (wx, w0[3] << 24, 3);
-              w1[0] = amd_bytealign (w1[0] >>  8, wx, 3);
-              break;
-    case 14:  w0[3] = amd_bytealign (wx, w0[3] << 16, 2);
-              w1[0] = amd_bytealign (w1[0] >> 16, wx, 2);
-              break;
-    case 15:  w0[3] = amd_bytealign (wx, w0[3] <<  8, 1);
-              w1[0] = amd_bytealign (w1[0] >> 24, wx, 1);
-              break;
-    case 16:  w1[0] = wx;
-              break;
-    case 17:  w1[0] = amd_bytealign (wx, w1[0] << 24, 3);
-              w1[1] = amd_bytealign (w1[1] >>  8, wx, 3);
-              break;
-    case 18:  w1[0] = amd_bytealign (wx, w1[0] << 16, 2);
-              w1[1] = amd_bytealign (w1[1] >> 16, wx, 2);
-              break;
-    case 19:  w1[0] = amd_bytealign (wx, w1[0] <<  8, 1);
-              w1[1] = amd_bytealign (w1[1] >> 24, wx, 1);
-              break;
-    case 20:  w1[1] = wx;
-              break;
-    case 21:  w1[1] = amd_bytealign (wx, w1[1] << 24, 3);
-              w1[2] = amd_bytealign (w1[2] >>  8, wx, 3);
-              break;
-    case 22:  w1[1] = amd_bytealign (wx, w1[1] << 16, 2);
-              w1[2] = amd_bytealign (w1[2] >> 16, wx, 2);
-              break;
-    case 23:  w1[1] = amd_bytealign (wx, w1[1] <<  8, 1);
-              w1[2] = amd_bytealign (w1[2] >> 24, wx, 1);
-              break;
-    case 24:  w1[2] = wx;
-              break;
-    case 25:  w1[2] = amd_bytealign (wx, w1[2] << 24, 3);
-              w1[3] = amd_bytealign (w1[3] >>  8, wx, 3);
-              break;
-    case 26:  w1[2] = amd_bytealign (wx, w1[2] << 16, 2);
-              w1[3] = amd_bytealign (w1[3] >> 16, wx, 2);
-              break;
-    case 27:  w1[2] = amd_bytealign (wx, w1[2] <<  8, 1);
-              w1[3] = amd_bytealign (w1[3] >> 24, wx, 1);
-              break;
-    case 28:  w1[3] = wx;
-              break;
-    case 29:  w1[3] = amd_bytealign (wx, w1[3] << 24, 3);
-              w2[0] = amd_bytealign (w2[0] >>  8, wx, 3);
-              break;
-    case 30:  w1[3] = amd_bytealign (wx, w1[3] << 16, 2);
-              w2[0] = amd_bytealign (w2[0] >> 16, wx, 2);
-              break;
-    case 31:  w1[3] = amd_bytealign (wx, w1[3] <<  8, 1);
-              w2[0] = amd_bytealign (w2[0] >> 24, wx, 1);
-              break;
-    case 32:  w2[0] = wx;
-              break;
-    case 33:  w2[0] = amd_bytealign (wx, w2[0] << 24, 3);
-              w2[1] = amd_bytealign (w2[1] >>  8, wx, 3);
-              break;
-    case 34:  w2[0] = amd_bytealign (wx, w2[0] << 16, 2);
-              w2[1] = amd_bytealign (w2[1] >> 16, wx, 2);
-              break;
-    case 35:  w2[0] = amd_bytealign (wx, w2[0] <<  8, 1);
-              w2[1] = amd_bytealign (w2[1] >> 24, wx, 1);
-              break;
-    case 36:  w2[1] = wx;
-              break;
-    case 37:  w2[1] = amd_bytealign (wx, w2[1] << 24, 3);
-              w2[2] = amd_bytealign (w2[2] >>  8, wx, 3);
-              break;
-    case 38:  w2[1] = amd_bytealign (wx, w2[1] << 16, 2);
-              w2[2] = amd_bytealign (w2[2] >> 16, wx, 2);
-              break;
-    case 39:  w2[1] = amd_bytealign (wx, w2[1] <<  8, 1);
-              w2[2] = amd_bytealign (w2[2] >> 24, wx, 1);
-              break;
-    case 40:  w2[2] = wx;
-              break;
-    case 41:  w2[2] = amd_bytealign (wx, w2[2] << 24, 3);
-              w2[3] = amd_bytealign (w2[3] >>  8, wx, 3);
-              break;
-    case 42:  w2[2] = amd_bytealign (wx, w2[2] << 16, 2);
-              w2[3] = amd_bytealign (w2[3] >> 16, wx, 2);
-              break;
-    case 43:  w2[2] = amd_bytealign (wx, w2[2] <<  8, 1);
-              w2[3] = amd_bytealign (w2[3] >> 24, wx, 1);
-              break;
-    case 44:  w2[3] = wx;
-              break;
-    case 45:  w2[3] = amd_bytealign (wx, w2[3] << 24, 3);
-              w3[0] = amd_bytealign (w3[0] >>  8, wx, 3);
-              break;
-    case 46:  w2[3] = amd_bytealign (wx, w2[3] << 16, 2);
-              w3[0] = amd_bytealign (w3[0] >> 16, wx, 2);
-              break;
-    case 47:  w2[3] = amd_bytealign (wx, w2[3] <<  8, 1);
-              w3[0] = amd_bytealign (w3[0] >> 24, wx, 1);
-              break;
-    case 48:  w3[0] = wx;
-              break;
-    case 49:  w3[0] = amd_bytealign (wx, w3[0] << 24, 3);
-              w3[1] = amd_bytealign (w3[1] >>  8, wx, 3);
-              break;
-    case 50:  w3[0] = amd_bytealign (wx, w3[0] << 16, 2);
-              w3[1] = amd_bytealign (w3[1] >> 16, wx, 2);
-              break;
-    case 51:  w3[0] = amd_bytealign (wx, w3[0] <<  8, 1);
-              w3[1] = amd_bytealign (w3[1] >> 24, wx, 1);
-              break;
-    case 52:  w3[1] = wx;
-              break;
-    case 53:  w3[1] = amd_bytealign (wx, w3[1] << 24, 3);
-              w3[2] = amd_bytealign (w3[2] >>  8, wx, 3);
-              break;
-    case 54:  w3[1] = amd_bytealign (wx, w3[1] << 16, 2);
-              w3[2] = amd_bytealign (w3[2] >> 16, wx, 2);
-              break;
-    case 55:  w3[1] = amd_bytealign (wx, w3[1] <<  8, 1);
-              w3[2] = amd_bytealign (w3[2] >> 24, wx, 1);
-              break;
-    case 56:  w3[2] = wx;
-              break;
-    case 57:  w3[2] = amd_bytealign (wx, w3[2] << 24, 3);
-              w3[3] = amd_bytealign (w3[3] >>  8, wx, 3);
-              break;
-    case 58:  w3[2] = amd_bytealign (wx, w3[2] << 16, 2);
-              w3[3] = amd_bytealign (w3[3] >> 16, wx, 2);
-              break;
-    case 59:  w3[2] = amd_bytealign (wx, w3[2] <<  8, 1);
-              w3[3] = amd_bytealign (w3[3] >> 24, wx, 1);
-              break;
-    case 60:  w3[3] = wx;
-              break;
-    case 61:  w3[3] = amd_bytealign (wx, w3[3] << 24, 3);
-              //w4[0] = amd_bytealign (w4[0] >>  8, wx, 3);
-              break;
-    case 62:  w3[3] = amd_bytealign (wx, w3[3] << 16, 2);
-              //w4[0] = amd_bytealign (w4[0] >> 16, wx, 2);
-              break;
-    case 63:  w3[3] = amd_bytealign (wx, w3[3] <<  8, 1);
-              //w4[0] = amd_bytealign (w4[0] >> 24, wx, 1);
-              break;
-  }
-  #else
-  switch (salt_len)
-  {
-    case  0:  w0[0] =  wx;
-              break;
-    case  1:  w0[0] = (w0[0] & 0x000000ff) | (wx <<  8);
-              w0[1] = (w0[1] & 0xffffff00) | (wx >> 24);
-              break;
-    case  2:  w0[0] = (w0[0] & 0x0000ffff) | (wx << 16);
-              w0[1] = (w0[1] & 0xffff0000) | (wx >> 16);
-              break;
-    case  3:  w0[0] = (w0[0] & 0x00ffffff) | (wx << 24);
-              w0[1] = (w0[1] & 0xff000000) | (wx >>  8);
-              break;
-    case  4:  w0[1] =  wx;
-              break;
-    case  5:  w0[1] = (w0[1] & 0x000000ff) | (wx <<  8);
-              w0[2] = (w0[2] & 0xffffff00) | (wx >> 24);
-              break;
-    case  6:  w0[1] = (w0[1] & 0x0000ffff) | (wx << 16);
-              w0[2] = (w0[2] & 0xffff0000) | (wx >> 16);
-              break;
-    case  7:  w0[1] = (w0[1] & 0x00ffffff) | (wx << 24);
-              w0[2] = (w0[2] & 0xff000000) | (wx >>  8);
-              break;
-    case  8:  w0[2] =  wx;
-              break;
-    case  9:  w0[2] = (w0[2] & 0x000000ff) | (wx <<  8);
-              w0[3] = (w0[3] & 0xffffff00) | (wx >> 24);
-              break;
-    case 10:  w0[2] = (w0[2] & 0x0000ffff) | (wx << 16);
-              w0[3] = (w0[3] & 0xffff0000) | (wx >> 16);
-              break;
-    case 11:  w0[2] = (w0[2] & 0x00ffffff) | (wx << 24);
-              w0[3] = (w0[3] & 0xff000000) | (wx >>  8);
-              break;
-    case 12:  w0[3] =  wx;
-              break;
-    case 13:  w0[3] = (w0[3] & 0x000000ff) | (wx <<  8);
-              w1[0] = (w1[0] & 0xffffff00) | (wx >> 24);
-              break;
-    case 14:  w0[3] = (w0[3] & 0x0000ffff) | (wx << 16);
-              w1[0] = (w1[0] & 0xffff0000) | (wx >> 16);
-              break;
-    case 15:  w0[3] = (w0[3] & 0x00ffffff) | (wx << 24);
-              w1[0] = (w1[0] & 0xff000000) | (wx >>  8);
-              break;
-    case 16:  w1[0] =  wx;
-              break;
-    case 17:  w1[0] = (w1[0] & 0x000000ff) | (wx <<  8);
-              w1[1] = (w1[1] & 0xffffff00) | (wx >> 24);
-              break;
-    case 18:  w1[0] = (w1[0] & 0x0000ffff) | (wx << 16);
-              w1[1] = (w1[1] & 0xffff0000) | (wx >> 16);
-              break;
-    case 19:  w1[0] = (w1[0] & 0x00ffffff) | (wx << 24);
-              w1[1] = (w1[1] & 0xff000000) | (wx >>  8);
-              break;
-    case 20:  w1[1] =  wx;
-              break;
-    case 21:  w1[1] = (w1[1] & 0x000000ff) | (wx <<  8);
-              w1[2] = (w1[2] & 0xffffff00) | (wx >> 24);
-              break;
-    case 22:  w1[1] = (w1[1] & 0x0000ffff) | (wx << 16);
-              w1[2] = (w1[2] & 0xffff0000) | (wx >> 16);
-              break;
-    case 23:  w1[1] = (w1[1] & 0x00ffffff) | (wx << 24);
-              w1[2] = (w1[2] & 0xff000000) | (wx >>  8);
-              break;
-    case 24:  w1[2] =  wx;
-              break;
-    case 25:  w1[2] = (w1[2] & 0x000000ff) | (wx <<  8);
-              w1[3] = (w1[3] & 0xffffff00) | (wx >> 24);
-              break;
-    case 26:  w1[2] = (w1[2] & 0x0000ffff) | (wx << 16);
-              w1[3] = (w1[3] & 0xffff0000) | (wx >> 16);
-              break;
-    case 27:  w1[2] = (w1[2] & 0x00ffffff) | (wx << 24);
-              w1[3] = (w1[3] & 0xff000000) | (wx >>  8);
-              break;
-    case 28:  w1[3] =  wx;
-              break;
-    case 29:  w1[3] = (w1[3] & 0x000000ff) | (wx <<  8);
-              w2[0] = (w2[0] & 0xffffff00) | (wx >> 24);
-              break;
-    case 30:  w1[3] = (w1[3] & 0x0000ffff) | (wx << 16);
-              w2[0] = (w2[0] & 0xffff0000) | (wx >> 16);
-              break;
-    case 31:  w1[3] = (w1[3] & 0x00ffffff) | (wx << 24);
-              w2[0] = (w2[0] & 0xff000000) | (wx >>  8);
-              break;
-    case 32:  w2[0] =  wx;
-              break;
-    case 33:  w2[0] = (w2[0] & 0x000000ff) | (wx <<  8);
-              w2[1] = (w2[1] & 0xffffff00) | (wx >> 24);
-              break;
-    case 34:  w2[0] = (w2[0] & 0x0000ffff) | (wx << 16);
-              w2[1] = (w2[1] & 0xffff0000) | (wx >> 16);
-              break;
-    case 35:  w2[0] = (w2[0] & 0x00ffffff) | (wx << 24);
-              w2[1] = (w2[1] & 0xff000000) | (wx >>  8);
-              break;
-    case 36:  w2[1] =  wx;
-              break;
-    case 37:  w2[1] = (w2[1] & 0x000000ff) | (wx <<  8);
-              w2[2] = (w2[2] & 0xffffff00) | (wx >> 24);
-              break;
-    case 38:  w2[1] = (w2[1] & 0x0000ffff) | (wx << 16);
-              w2[2] = (w2[2] & 0xffff0000) | (wx >> 16);
-              break;
-    case 39:  w2[1] = (w2[1] & 0x00ffffff) | (wx << 24);
-              w2[2] = (w2[2] & 0xff000000) | (wx >>  8);
-              break;
-    case 40:  w2[2] =  wx;
-              break;
-    case 41:  w2[2] = (w2[2] & 0x000000ff) | (wx <<  8);
-              w2[3] = (w2[3] & 0xffffff00) | (wx >> 24);
-              break;
-    case 42:  w2[2] = (w2[2] & 0x0000ffff) | (wx << 16);
-              w2[3] = (w2[3] & 0xffff0000) | (wx >> 16);
-              break;
-    case 43:  w2[2] = (w2[2] & 0x00ffffff) | (wx << 24);
-              w2[3] = (w2[3] & 0xff000000) | (wx >>  8);
-              break;
-    case 44:  w2[3] =  wx;
-              break;
-    case 45:  w2[3] = (w2[3] & 0x000000ff) | (wx <<  8);
-              w3[0] = (w3[0] & 0xffffff00) | (wx >> 24);
-              break;
-    case 46:  w2[3] = (w2[3] & 0x0000ffff) | (wx << 16);
-              w3[0] = (w3[0] & 0xffff0000) | (wx >> 16);
-              break;
-    case 47:  w2[3] = (w2[3] & 0x00ffffff) | (wx << 24);
-              w3[0] = (w3[0] & 0xff000000) | (wx >>  8);
-              break;
-    case 48:  w3[0] =  wx;
-              break;
-    case 49:  w3[0] = (w3[0] & 0x000000ff) | (wx <<  8);
-              w3[1] = (w3[1] & 0xffffff00) | (wx >> 24);
-              break;
-    case 50:  w3[0] = (w3[0] & 0x0000ffff) | (wx << 16);
-              w3[1] = (w3[1] & 0xffff0000) | (wx >> 16);
-              break;
-    case 51:  w3[0] = (w3[0] & 0x00ffffff) | (wx << 24);
-              w3[1] = (w3[1] & 0xff000000) | (wx >>  8);
-              break;
-    case 52:  w3[1] =  wx;
-              break;
-    case 53:  w3[1] = (w3[1] & 0x000000ff) | (wx <<  8);
-              w3[2] = (w3[2] & 0xffffff00) | (wx >> 24);
-              break;
-    case 54:  w3[1] = (w3[1] & 0x0000ffff) | (wx << 16);
-              w3[2] = (w3[2] & 0xffff0000) | (wx >> 16);
-              break;
-    case 55:  w3[1] = (w3[1] & 0x00ffffff) | (wx << 24);
-              w3[2] = (w3[2] & 0xff000000) | (wx >>  8);
-              break;
-    case 56:  w3[2] =  wx;
-              break;
-    case 57:  w3[2] = (w3[2] & 0x000000ff) | (wx <<  8);
-              w3[3] = (w3[3] & 0xffffff00) | (wx >> 24);
-              break;
-    case 58:  w3[2] = (w3[2] & 0x0000ffff) | (wx << 16);
-              w3[3] = (w3[3] & 0xffff0000) | (wx >> 16);
-              break;
-    case 59:  w3[2] = (w3[2] & 0x00ffffff) | (wx << 24);
-              w3[3] = (w3[3] & 0xff000000) | (wx >>  8);
-              break;
-    case 60:  w3[3] =  wx;
-              break;
-    case 61:  w3[3] = (w3[3] & 0x000000ff) | (wx <<  8);
-              //w4[0] = (w4[0] & 0xffffff00) | (wx >> 24);
-              break;
-    case 62:  w3[3] = (w3[3] & 0x0000ffff) | (wx << 16);
-              //w4[0] = (w4[0] & 0xffff0000) | (wx >> 16);
-              break;
-    case 63:  w3[3] = (w3[3] & 0x00ffffff) | (wx << 24);
-              //w4[0] = (w4[0] & 0xff000000) | (wx >>  8);
-              break;
-  }
-  #endif
-}
-
-inline void overwrite_at_be_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x wx, const u32 salt_len)
-{
-  // would be nice to have optimization based on amd_bytealign as with _le counterpart
-
-  switch (salt_len)
-  {
-    case  0:  w0[0] =  wx;
-              break;
-    case  1:  w0[0] = (w0[0] & 0xff000000) | (wx >>  8);
-              w0[1] = (w0[1] & 0x00ffffff) | (wx << 24);
-              break;
-    case  2:  w0[0] = (w0[0] & 0xffff0000) | (wx >> 16);
-              w0[1] = (w0[1] & 0x0000ffff) | (wx << 16);
-              break;
-    case  3:  w0[0] = (w0[0] & 0xffffff00) | (wx >> 24);
-              w0[1] = (w0[1] & 0x000000ff) | (wx <<  8);
-              break;
-    case  4:  w0[1] =  wx;
-              break;
-    case  5:  w0[1] = (w0[1] & 0xff000000) | (wx >>  8);
-              w0[2] = (w0[2] & 0x00ffffff) | (wx << 24);
-              break;
-    case  6:  w0[1] = (w0[1] & 0xffff0000) | (wx >> 16);
-              w0[2] = (w0[2] & 0x0000ffff) | (wx << 16);
-              break;
-    case  7:  w0[1] = (w0[1] & 0xffffff00) | (wx >> 24);
-              w0[2] = (w0[2] & 0x000000ff) | (wx <<  8);
-              break;
-    case  8:  w0[2] =  wx;
-              break;
-    case  9:  w0[2] = (w0[2] & 0xff000000) | (wx >>  8);
-              w0[3] = (w0[3] & 0x00ffffff) | (wx << 24);
-              break;
-    case 10:  w0[2] = (w0[2] & 0xffff0000) | (wx >> 16);
-              w0[3] = (w0[3] & 0x0000ffff) | (wx << 16);
-              break;
-    case 11:  w0[2] = (w0[2] & 0xffffff00) | (wx >> 24);
-              w0[3] = (w0[3] & 0x000000ff) | (wx <<  8);
-              break;
-    case 12:  w0[3] =  wx;
-              break;
-    case 13:  w0[3] = (w0[3] & 0xff000000) | (wx >>  8);
-              w1[0] = (w1[0] & 0x00ffffff) | (wx << 24);
-              break;
-    case 14:  w0[3] = (w0[3] & 0xffff0000) | (wx >> 16);
-              w1[0] = (w1[0] & 0x0000ffff) | (wx << 16);
-              break;
-    case 15:  w0[3] = (w0[3] & 0xffffff00) | (wx >> 24);
-              w1[0] = (w1[0] & 0x000000ff) | (wx <<  8);
-              break;
-    case 16:  w1[0] =  wx;
-              break;
-    case 17:  w1[0] = (w1[0] & 0xff000000) | (wx >>  8);
-              w1[1] = (w1[1] & 0x00ffffff) | (wx << 24);
-              break;
-    case 18:  w1[0] = (w1[0] & 0xffff0000) | (wx >> 16);
-              w1[1] = (w1[1] & 0x0000ffff) | (wx << 16);
-              break;
-    case 19:  w1[0] = (w1[0] & 0xffffff00) | (wx >> 24);
-              w1[1] = (w1[1] & 0x000000ff) | (wx <<  8);
-              break;
-    case 20:  w1[1] =  wx;
-              break;
-    case 21:  w1[1] = (w1[1] & 0xff000000) | (wx >>  8);
-              w1[2] = (w1[2] & 0x00ffffff) | (wx << 24);
-              break;
-    case 22:  w1[1] = (w1[1] & 0xffff0000) | (wx >> 16);
-              w1[2] = (w1[2] & 0x0000ffff) | (wx << 16);
-              break;
-    case 23:  w1[1] = (w1[1] & 0xffffff00) | (wx >> 24);
-              w1[2] = (w1[2] & 0x000000ff) | (wx <<  8);
-              break;
-    case 24:  w1[2] =  wx;
-              break;
-    case 25:  w1[2] = (w1[2] & 0xff000000) | (wx >>  8);
-              w1[3] = (w1[3] & 0x00ffffff) | (wx << 24);
-              break;
-    case 26:  w1[2] = (w1[2] & 0xffff0000) | (wx >> 16);
-              w1[3] = (w1[3] & 0x0000ffff) | (wx << 16);
-              break;
-    case 27:  w1[2] = (w1[2] & 0xffffff00) | (wx >> 24);
-              w1[3] = (w1[3] & 0x000000ff) | (wx <<  8);
-              break;
-    case 28:  w1[3] =  wx;
-              break;
-    case 29:  w1[3] = (w1[3] & 0xff000000) | (wx >>  8);
-              w2[0] = (w2[0] & 0x00ffffff) | (wx << 24);
-              break;
-    case 30:  w1[3] = (w1[3] & 0xffff0000) | (wx >> 16);
-              w2[0] = (w2[0] & 0x0000ffff) | (wx << 16);
-              break;
-    case 31:  w1[3] = (w1[3] & 0xffffff00) | (wx >> 24);
-              w2[0] = (w2[0] & 0x000000ff) | (wx <<  8);
-              break;
-    case 32:  w2[0] =  wx;
-              break;
-    case 33:  w2[0] = (w2[0] & 0xff000000) | (wx >>  8);
-              w2[1] = (w2[1] & 0x00ffffff) | (wx << 24);
-              break;
-    case 34:  w2[0] = (w2[0] & 0xffff0000) | (wx >> 16);
-              w2[1] = (w2[1] & 0x0000ffff) | (wx << 16);
-              break;
-    case 35:  w2[0] = (w2[0] & 0xffffff00) | (wx >> 24);
-              w2[1] = (w2[1] & 0x000000ff) | (wx <<  8);
-              break;
-    case 36:  w2[1] =  wx;
-              break;
-    case 37:  w2[1] = (w2[1] & 0xff000000) | (wx >>  8);
-              w2[2] = (w2[2] & 0x00ffffff) | (wx << 24);
-              break;
-    case 38:  w2[1] = (w2[1] & 0xffff0000) | (wx >> 16);
-              w2[2] = (w2[2] & 0x0000ffff) | (wx << 16);
-              break;
-    case 39:  w2[1] = (w2[1] & 0xffffff00) | (wx >> 24);
-              w2[2] = (w2[2] & 0x000000ff) | (wx <<  8);
-              break;
-    case 40:  w2[2] =  wx;
-              break;
-    case 41:  w2[2] = (w2[2] & 0xff000000) | (wx >>  8);
-              w2[3] = (w2[3] & 0x00ffffff) | (wx << 24);
-              break;
-    case 42:  w2[2] = (w2[2] & 0xffff0000) | (wx >> 16);
-              w2[3] = (w2[3] & 0x0000ffff) | (wx << 16);
-              break;
-    case 43:  w2[2] = (w2[2] & 0xffffff00) | (wx >> 24);
-              w2[3] = (w2[3] & 0x000000ff) | (wx <<  8);
-              break;
-    case 44:  w2[3] =  wx;
-              break;
-    case 45:  w2[3] = (w2[3] & 0xff000000) | (wx >>  8);
-              w3[0] = (w3[0] & 0x00ffffff) | (wx << 24);
-              break;
-    case 46:  w2[3] = (w2[3] & 0xffff0000) | (wx >> 16);
-              w3[0] = (w3[0] & 0x0000ffff) | (wx << 16);
-              break;
-    case 47:  w2[3] = (w2[3] & 0xffffff00) | (wx >> 24);
-              w3[0] = (w3[0] & 0x000000ff) | (wx <<  8);
-              break;
-    case 48:  w3[0] =  wx;
-              break;
-    case 49:  w3[0] = (w3[0] & 0xff000000) | (wx >>  8);
-              w3[1] = (w3[1] & 0x00ffffff) | (wx << 24);
-              break;
-    case 50:  w3[0] = (w3[0] & 0xffff0000) | (wx >> 16);
-              w3[1] = (w3[1] & 0x0000ffff) | (wx << 16);
-              break;
-    case 51:  w3[0] = (w3[0] & 0xffffff00) | (wx >> 24);
-              w3[1] = (w3[1] & 0x000000ff) | (wx <<  8);
-              break;
-    case 52:  w3[1] =  wx;
-              break;
-    case 53:  w3[1] = (w3[1] & 0xff000000) | (wx >>  8);
-              w3[2] = (w3[2] & 0x00ffffff) | (wx << 24);
-              break;
-    case 54:  w3[1] = (w3[1] & 0xffff0000) | (wx >> 16);
-              w3[2] = (w3[2] & 0x0000ffff) | (wx << 16);
-              break;
-    case 55:  w3[1] = (w3[1] & 0xffffff00) | (wx >> 24);
-              w3[2] = (w3[2] & 0x000000ff) | (wx <<  8);
-              break;
-    case 56:  w3[2] =  wx;
-              break;
-    case 57:  w3[2] = (w3[2] & 0xff000000) | (wx >>  8);
-              w3[3] = (w3[3] & 0x00ffffff) | (wx << 24);
-              break;
-    case 58:  w3[2] = (w3[2] & 0xffff0000) | (wx >> 16);
-              w3[3] = (w3[3] & 0x0000ffff) | (wx << 16);
-              break;
-    case 59:  w3[2] = (w3[2] & 0xffffff00) | (wx >> 24);
-              w3[3] = (w3[3] & 0x000000ff) | (wx <<  8);
-              break;
-    case 60:  w3[3] =  wx;
-              break;
-    case 61:  w3[3] = (w3[3] & 0xff000000) | (wx >>  8);
-              //w4[0] = (w4[0] & 0x00ffffff) | (wx << 24);
-              break;
-    case 62:  w3[3] = (w3[3] & 0xffff0000) | (wx >> 16);
-              //w4[0] = (w4[0] & 0x0000ffff) | (wx << 16);
-              break;
-    case 63:  w3[3] = (w3[3] & 0xffffff00) | (wx >> 24);
-              //w4[0] = (w4[0] & 0x000000ff) | (wx <<  8);
-              break;
-  }
-}
-
-/**
- * vector functions as scalar (for outer loop usage)
- */
-
-inline void append_0x01_1x4_S (u32 w0[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0]  = 0x01;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0100;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x010000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x01000000;
-      break;
-
-    case 4:
-      w0[1] = 0x01;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0100;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x010000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x01000000;
-      break;
-
-    case 8:
-      w0[2] = 0x01;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0100;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x010000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x01000000;
-      break;
-
-    case 12:
-      w0[3] = 0x01;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0100;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x010000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x01000000;
-      break;
-  }
-}
-
-inline void append_0x01_2x4_S (u32 w0[4], u32 w1[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x01;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0100;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x010000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x01000000;
-      break;
-
-    case 4:
-      w0[1] = 0x01;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0100;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x010000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x01000000;
-      break;
-
-    case 8:
-      w0[2] = 0x01;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0100;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x010000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x01000000;
-      break;
-
-    case 12:
-      w0[3] = 0x01;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0100;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x010000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x01000000;
-      break;
-
-    case 16:
-      w1[0] = 0x01;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0100;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x010000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x01000000;
-      break;
-
-    case 20:
-      w1[1] = 0x01;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0100;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x010000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x01000000;
-      break;
-
-    case 24:
-      w1[2] = 0x01;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0100;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x010000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x01000000;
-      break;
-
-    case 28:
-      w1[3] = 0x01;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0100;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x010000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x01000000;
-      break;
-  }
-}
-
-inline void append_0x01_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x01;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0100;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x010000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x01000000;
-      break;
-
-    case 4:
-      w0[1] = 0x01;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0100;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x010000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x01000000;
-      break;
-
-    case 8:
-      w0[2] = 0x01;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0100;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x010000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x01000000;
-      break;
-
-    case 12:
-      w0[3] = 0x01;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0100;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x010000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x01000000;
-      break;
-
-    case 16:
-      w1[0] = 0x01;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0100;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x010000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x01000000;
-      break;
-
-    case 20:
-      w1[1] = 0x01;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0100;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x010000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x01000000;
-      break;
-
-    case 24:
-      w1[2] = 0x01;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0100;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x010000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x01000000;
-      break;
-
-    case 28:
-      w1[3] = 0x01;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0100;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x010000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x01000000;
-      break;
-
-    case 32:
-      w2[0] = 0x01;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x0100;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x010000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x01000000;
-      break;
-
-    case 36:
-      w2[1] = 0x01;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x0100;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x010000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x01000000;
-      break;
-
-    case 40:
-      w2[2] = 0x01;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x0100;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x010000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x01000000;
-      break;
-
-    case 44:
-      w2[3] = 0x01;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x0100;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x010000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x01000000;
-      break;
-  }
-}
-
-inline void append_0x01_4x4_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x01;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0100;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x010000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x01000000;
-      break;
-
-    case 4:
-      w0[1] = 0x01;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0100;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x010000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x01000000;
-      break;
-
-    case 8:
-      w0[2] = 0x01;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0100;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x010000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x01000000;
-      break;
-
-    case 12:
-      w0[3] = 0x01;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0100;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x010000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x01000000;
-      break;
-
-    case 16:
-      w1[0] = 0x01;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0100;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x010000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x01000000;
-      break;
-
-    case 20:
-      w1[1] = 0x01;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0100;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x010000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x01000000;
-      break;
-
-    case 24:
-      w1[2] = 0x01;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0100;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x010000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x01000000;
-      break;
-
-    case 28:
-      w1[3] = 0x01;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0100;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x010000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x01000000;
-      break;
-
-    case 32:
-      w2[0] = 0x01;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x0100;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x010000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x01000000;
-      break;
-
-    case 36:
-      w2[1] = 0x01;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x0100;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x010000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x01000000;
-      break;
-
-    case 40:
-      w2[2] = 0x01;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x0100;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x010000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x01000000;
-      break;
-
-    case 44:
-      w2[3] = 0x01;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x0100;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x010000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x01000000;
-      break;
-
-    case 48:
-      w3[0] = 0x01;
-      break;
-
-    case 49:
-      w3[0] = w3[0] | 0x0100;
-      break;
-
-    case 50:
-      w3[0] = w3[0] | 0x010000;
-      break;
-
-    case 51:
-      w3[0] = w3[0] | 0x01000000;
-      break;
-
-    case 52:
-      w3[1] = 0x01;
-      break;
-
-    case 53:
-      w3[1] = w3[1] | 0x0100;
-      break;
-
-    case 54:
-      w3[1] = w3[1] | 0x010000;
-      break;
-
-    case 55:
-      w3[1] = w3[1] | 0x01000000;
-      break;
-
-    case 56:
-      w3[2] = 0x01;
-      break;
-
-    case 57:
-      w3[2] = w3[2] | 0x0100;
-      break;
-
-    case 58:
-      w3[2] = w3[2] | 0x010000;
-      break;
-
-    case 59:
-      w3[2] = w3[2] | 0x01000000;
-      break;
-
-    case 60:
-      w3[3] = 0x01;
-      break;
-
-    case 61:
-      w3[3] = w3[3] | 0x0100;
-      break;
-
-    case 62:
-      w3[3] = w3[3] | 0x010000;
-      break;
-
-    case 63:
-      w3[3] = w3[3] | 0x01000000;
-      break;
-  }
-}
-
-inline void append_0x02_2x4_S (u32 w0[4], u32 w1[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x02;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0200;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x020000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x02000000;
-      break;
-
-    case 4:
-      w0[1] = 0x02;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0200;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x020000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x02000000;
-      break;
-
-    case 8:
-      w0[2] = 0x02;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0200;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x020000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x02000000;
-      break;
-
-    case 12:
-      w0[3] = 0x02;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0200;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x020000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x02000000;
-      break;
-
-    case 16:
-      w1[0] = 0x02;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0200;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x020000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x02000000;
-      break;
-
-    case 20:
-      w1[1] = 0x02;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0200;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x020000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x02000000;
-      break;
-
-    case 24:
-      w1[2] = 0x02;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0200;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x020000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x02000000;
-      break;
-
-    case 28:
-      w1[3] = 0x02;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0200;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x020000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x02000000;
-      break;
-  }
-}
-
-inline void append_0x02_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x02;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x0200;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x020000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x02000000;
-      break;
-
-    case 4:
-      w0[1] = 0x02;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x0200;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x020000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x02000000;
-      break;
-
-    case 8:
-      w0[2] = 0x02;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x0200;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x020000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x02000000;
-      break;
-
-    case 12:
-      w0[3] = 0x02;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x0200;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x020000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x02000000;
-      break;
-
-    case 16:
-      w1[0] = 0x02;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x0200;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x020000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x02000000;
-      break;
-
-    case 20:
-      w1[1] = 0x02;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x0200;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x020000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x02000000;
-      break;
-
-    case 24:
-      w1[2] = 0x02;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x0200;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x020000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x02000000;
-      break;
-
-    case 28:
-      w1[3] = 0x02;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x0200;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x020000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x02000000;
-      break;
-
-    case 32:
-      w2[0] = 0x02;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x0200;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x020000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x02000000;
-      break;
-
-    case 36:
-      w2[1] = 0x02;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x0200;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x020000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x02000000;
-      break;
-
-    case 40:
-      w2[2] = 0x02;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x0200;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x020000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x02000000;
-      break;
-
-    case 44:
-      w2[3] = 0x02;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x0200;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x020000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x02000000;
-      break;
-  }
-}
-
-inline void append_0x80_1x4_S (u32 w0[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0]  = 0x80;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x8000;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x800000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x80000000;
-      break;
-
-    case 4:
-      w0[1] = 0x80;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x8000;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x800000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x80000000;
-      break;
-
-    case 8:
-      w0[2] = 0x80;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x8000;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x800000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x80000000;
-      break;
-
-    case 12:
-      w0[3] = 0x80;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x8000;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x800000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x80000000;
-      break;
-  }
-}
-
-inline void append_0x80_2x4_S (u32 w0[4], u32 w1[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x80;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x8000;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x800000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x80000000;
-      break;
-
-    case 4:
-      w0[1] = 0x80;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x8000;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x800000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x80000000;
-      break;
-
-    case 8:
-      w0[2] = 0x80;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x8000;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x800000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x80000000;
-      break;
-
-    case 12:
-      w0[3] = 0x80;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x8000;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x800000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x80000000;
-      break;
-
-    case 16:
-      w1[0] = 0x80;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x8000;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x800000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x80000000;
-      break;
-
-    case 20:
-      w1[1] = 0x80;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x8000;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x800000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x80000000;
-      break;
-
-    case 24:
-      w1[2] = 0x80;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x8000;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x800000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x80000000;
-      break;
-
-    case 28:
-      w1[3] = 0x80;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x8000;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x800000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x80000000;
-      break;
-  }
-}
-
-inline void append_0x80_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x80;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x8000;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x800000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x80000000;
-      break;
-
-    case 4:
-      w0[1] = 0x80;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x8000;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x800000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x80000000;
-      break;
-
-    case 8:
-      w0[2] = 0x80;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x8000;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x800000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x80000000;
-      break;
-
-    case 12:
-      w0[3] = 0x80;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x8000;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x800000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x80000000;
-      break;
-
-    case 16:
-      w1[0] = 0x80;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x8000;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x800000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x80000000;
-      break;
-
-    case 20:
-      w1[1] = 0x80;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x8000;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x800000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x80000000;
-      break;
-
-    case 24:
-      w1[2] = 0x80;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x8000;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x800000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x80000000;
-      break;
-
-    case 28:
-      w1[3] = 0x80;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x8000;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x800000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x80000000;
-      break;
-
-    case 32:
-      w2[0] = 0x80;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x8000;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x800000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x80000000;
-      break;
-
-    case 36:
-      w2[1] = 0x80;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x8000;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x800000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x80000000;
-      break;
-
-    case 40:
-      w2[2] = 0x80;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x8000;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x800000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x80000000;
-      break;
-
-    case 44:
-      w2[3] = 0x80;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x8000;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x800000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x80000000;
-      break;
-  }
-}
-
-inline void append_0x80_4x4_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset)
-{
-  switch (offset)
-  {
-    case 0:
-      w0[0] = 0x80;
-      break;
-
-    case 1:
-      w0[0] = w0[0] | 0x8000;
-      break;
-
-    case 2:
-      w0[0] = w0[0] | 0x800000;
-      break;
-
-    case 3:
-      w0[0] = w0[0] | 0x80000000;
-      break;
-
-    case 4:
-      w0[1] = 0x80;
-      break;
-
-    case 5:
-      w0[1] = w0[1] | 0x8000;
-      break;
-
-    case 6:
-      w0[1] = w0[1] | 0x800000;
-      break;
-
-    case 7:
-      w0[1] = w0[1] | 0x80000000;
-      break;
-
-    case 8:
-      w0[2] = 0x80;
-      break;
-
-    case 9:
-      w0[2] = w0[2] | 0x8000;
-      break;
-
-    case 10:
-      w0[2] = w0[2] | 0x800000;
-      break;
-
-    case 11:
-      w0[2] = w0[2] | 0x80000000;
-      break;
-
-    case 12:
-      w0[3] = 0x80;
-      break;
-
-    case 13:
-      w0[3] = w0[3] | 0x8000;
-      break;
-
-    case 14:
-      w0[3] = w0[3] | 0x800000;
-      break;
-
-    case 15:
-      w0[3] = w0[3] | 0x80000000;
-      break;
-
-    case 16:
-      w1[0] = 0x80;
-      break;
-
-    case 17:
-      w1[0] = w1[0] | 0x8000;
-      break;
-
-    case 18:
-      w1[0] = w1[0] | 0x800000;
-      break;
-
-    case 19:
-      w1[0] = w1[0] | 0x80000000;
-      break;
-
-    case 20:
-      w1[1] = 0x80;
-      break;
-
-    case 21:
-      w1[1] = w1[1] | 0x8000;
-      break;
-
-    case 22:
-      w1[1] = w1[1] | 0x800000;
-      break;
-
-    case 23:
-      w1[1] = w1[1] | 0x80000000;
-      break;
-
-    case 24:
-      w1[2] = 0x80;
-      break;
-
-    case 25:
-      w1[2] = w1[2] | 0x8000;
-      break;
-
-    case 26:
-      w1[2] = w1[2] | 0x800000;
-      break;
-
-    case 27:
-      w1[2] = w1[2] | 0x80000000;
-      break;
-
-    case 28:
-      w1[3] = 0x80;
-      break;
-
-    case 29:
-      w1[3] = w1[3] | 0x8000;
-      break;
-
-    case 30:
-      w1[3] = w1[3] | 0x800000;
-      break;
-
-    case 31:
-      w1[3] = w1[3] | 0x80000000;
-      break;
-
-    case 32:
-      w2[0] = 0x80;
-      break;
-
-    case 33:
-      w2[0] = w2[0] | 0x8000;
-      break;
-
-    case 34:
-      w2[0] = w2[0] | 0x800000;
-      break;
-
-    case 35:
-      w2[0] = w2[0] | 0x80000000;
-      break;
-
-    case 36:
-      w2[1] = 0x80;
-      break;
-
-    case 37:
-      w2[1] = w2[1] | 0x8000;
-      break;
-
-    case 38:
-      w2[1] = w2[1] | 0x800000;
-      break;
-
-    case 39:
-      w2[1] = w2[1] | 0x80000000;
-      break;
-
-    case 40:
-      w2[2] = 0x80;
-      break;
-
-    case 41:
-      w2[2] = w2[2] | 0x8000;
-      break;
-
-    case 42:
-      w2[2] = w2[2] | 0x800000;
-      break;
-
-    case 43:
-      w2[2] = w2[2] | 0x80000000;
-      break;
-
-    case 44:
-      w2[3] = 0x80;
-      break;
-
-    case 45:
-      w2[3] = w2[3] | 0x8000;
-      break;
-
-    case 46:
-      w2[3] = w2[3] | 0x800000;
-      break;
-
-    case 47:
-      w2[3] = w2[3] | 0x80000000;
-      break;
-
-    case 48:
-      w3[0] = 0x80;
-      break;
-
-    case 49:
-      w3[0] = w3[0] | 0x8000;
-      break;
-
-    case 50:
-      w3[0] = w3[0] | 0x800000;
-      break;
-
-    case 51:
-      w3[0] = w3[0] | 0x80000000;
-      break;
-
-    case 52:
-      w3[1] = 0x80;
-      break;
-
-    case 53:
-      w3[1] = w3[1] | 0x8000;
-      break;
-
-    case 54:
-      w3[1] = w3[1] | 0x800000;
-      break;
-
-    case 55:
-      w3[1] = w3[1] | 0x80000000;
-      break;
-
-    case 56:
-      w3[2] = 0x80;
-      break;
-
-    case 57:
-      w3[2] = w3[2] | 0x8000;
-      break;
-
-    case 58:
-      w3[2] = w3[2] | 0x800000;
-      break;
-
-    case 59:
-      w3[2] = w3[2] | 0x80000000;
-      break;
-
-    case 60:
-      w3[3] = 0x80;
-      break;
-
-    case 61:
-      w3[3] = w3[3] | 0x8000;
-      break;
-
-    case 62:
-      w3[3] = w3[3] | 0x800000;
-      break;
-
-    case 63:
-      w3[3] = w3[3] | 0x80000000;
-      break;
-  }
-}
-
-inline void truncate_block_S (u32 w[4], const u32 len)
-{
-  switch (len)
-  {
-    case  0:  w[0] &= 0;
-              w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  1:  w[0] &= 0x000000FF;
-              w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  2:  w[0] &= 0x0000FFFF;
-              w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  3:  w[0] &= 0x00FFFFFF;
-              w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  4:  w[1] &= 0;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  5:  w[1] &= 0x000000FF;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  6:  w[1] &= 0x0000FFFF;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  7:  w[1] &= 0x00FFFFFF;
-              w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  8:  w[2] &= 0;
-              w[3] &= 0;
-              break;
-    case  9:  w[2] &= 0x000000FF;
-              w[3] &= 0;
-              break;
-    case 10:  w[2] &= 0x0000FFFF;
-              w[3] &= 0;
-              break;
-    case 11:  w[2] &= 0x00FFFFFF;
-              w[3] &= 0;
-              break;
-    case 12:  w[3] &= 0;
-              break;
-    case 13:  w[3] &= 0x000000FF;
-              break;
-    case 14:  w[3] &= 0x0000FFFF;
-              break;
-    case 15:  w[3] &= 0x00FFFFFF;
-              break;
-  }
-}
-
-inline void make_unicode_S (const u32 in[4], u32 out1[4], u32 out2[4])
-{
-  #ifdef IS_NV
-  out2[3] = __byte_perm_S (in[3], 0, 0x7372);
-  out2[2] = __byte_perm_S (in[3], 0, 0x7170);
-  out2[1] = __byte_perm_S (in[2], 0, 0x7372);
-  out2[0] = __byte_perm_S (in[2], 0, 0x7170);
-  out1[3] = __byte_perm_S (in[1], 0, 0x7372);
-  out1[2] = __byte_perm_S (in[1], 0, 0x7170);
-  out1[1] = __byte_perm_S (in[0], 0, 0x7372);
-  out1[0] = __byte_perm_S (in[0], 0, 0x7170);
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  out2[3]  = ((in[3] >> 8) & 0x00FF0000) | ((in[3] >> 16) & 0x000000FF);
-  out2[2]  = ((in[3] << 8) & 0x00FF0000) | ((in[3] >>  0) & 0x000000FF);
-  out2[1]  = ((in[2] >> 8) & 0x00FF0000) | ((in[2] >> 16) & 0x000000FF);
-  out2[0]  = ((in[2] << 8) & 0x00FF0000) | ((in[2] >>  0) & 0x000000FF);
-  out1[3]  = ((in[1] >> 8) & 0x00FF0000) | ((in[1] >> 16) & 0x000000FF);
-  out1[2]  = ((in[1] << 8) & 0x00FF0000) | ((in[1] >>  0) & 0x000000FF);
-  out1[1]  = ((in[0] >> 8) & 0x00FF0000) | ((in[0] >> 16) & 0x000000FF);
-  out1[0]  = ((in[0] << 8) & 0x00FF0000) | ((in[0] >>  0) & 0x000000FF);
-  #endif
-}
-
-inline void undo_unicode_S (const u32 in1[4], const u32 in2[4], u32 out[4])
-{
-  #ifdef IS_NV
-  out[0] = __byte_perm_S (in1[0], in1[1], 0x6420);
-  out[1] = __byte_perm_S (in1[2], in1[3], 0x6420);
-  out[2] = __byte_perm_S (in2[0], in2[1], 0x6420);
-  out[3] = __byte_perm_S (in2[2], in2[3], 0x6420);
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  out[0] = ((in1[0] & 0x000000ff) >>  0) | ((in1[0] & 0x00ff0000) >>  8)
-         | ((in1[1] & 0x000000ff) << 16) | ((in1[1] & 0x00ff0000) <<  8);
-  out[1] = ((in1[2] & 0x000000ff) >>  0) | ((in1[2] & 0x00ff0000) >>  8)
-         | ((in1[3] & 0x000000ff) << 16) | ((in1[3] & 0x00ff0000) <<  8);
-  out[2] = ((in2[0] & 0x000000ff) >>  0) | ((in2[0] & 0x00ff0000) >>  8)
-         | ((in2[1] & 0x000000ff) << 16) | ((in2[1] & 0x00ff0000) <<  8);
-  out[3] = ((in2[2] & 0x000000ff) >>  0) | ((in2[2] & 0x00ff0000) >>  8)
-         | ((in2[3] & 0x000000ff) << 16) | ((in2[3] & 0x00ff0000) <<  8);
-  #endif
-}
-
-inline void switch_buffer_by_offset_le_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset)
-{
-  #if defined IS_AMD || defined IS_GENERIC
-  const int offset_mod_4 = offset & 3;
-
-  const int offset_minus_4 = 4 - offset;
-
-  switch (offset / 4)
-  {
-    case 0:
-      w3[2] = amd_bytealign_S (    0, w3[1], offset_minus_4);
-      w3[1] = amd_bytealign_S (w3[1], w3[0], offset_minus_4);
-      w3[0] = amd_bytealign_S (w3[0], w2[3], offset_minus_4);
-      w2[3] = amd_bytealign_S (w2[3], w2[2], offset_minus_4);
-      w2[2] = amd_bytealign_S (w2[2], w2[1], offset_minus_4);
-      w2[1] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
-      w2[0] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
-      w1[3] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
-      w1[2] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
-      w1[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
-      w1[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w0[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w0[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w0[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w0[0] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-
-      if (offset_mod_4 == 0)
-      {
-        w0[0] = w0[1];
-        w0[1] = w0[2];
-        w0[2] = w0[3];
-        w0[3] = w1[0];
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 1:
-      w3[2] = amd_bytealign_S (    0, w3[0], offset_minus_4);
-      w3[1] = amd_bytealign_S (w3[0], w2[3], offset_minus_4);
-      w3[0] = amd_bytealign_S (w2[3], w2[2], offset_minus_4);
-      w2[3] = amd_bytealign_S (w2[2], w2[1], offset_minus_4);
-      w2[2] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
-      w2[1] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
-      w2[0] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
-      w1[3] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
-      w1[2] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
-      w1[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w1[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w0[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w0[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w0[1] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w0[1] = w0[2];
-        w0[2] = w0[3];
-        w0[3] = w1[0];
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 2:
-      w3[2] = amd_bytealign_S (    0, w2[3], offset_minus_4);
-      w3[1] = amd_bytealign_S (w2[3], w2[2], offset_minus_4);
-      w3[0] = amd_bytealign_S (w2[2], w2[1], offset_minus_4);
-      w2[3] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
-      w2[2] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
-      w2[1] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
-      w2[0] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
-      w1[3] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
-      w1[2] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w1[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w1[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w0[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w0[2] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w0[2] = w0[3];
-        w0[3] = w1[0];
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 3:
-      w3[2] = amd_bytealign_S (    0, w2[2], offset_minus_4);
-      w3[1] = amd_bytealign_S (w2[2], w2[1], offset_minus_4);
-      w3[0] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
-      w2[3] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
-      w2[2] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
-      w2[1] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
-      w2[0] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
-      w1[3] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w1[2] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w1[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w1[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w0[3] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w0[3] = w1[0];
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 4:
-      w3[2] = amd_bytealign_S (    0, w2[1], offset_minus_4);
-      w3[1] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
-      w3[0] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
-      w2[3] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
-      w2[2] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
-      w2[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
-      w2[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w1[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w1[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w1[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w1[0] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w1[0] = w1[1];
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 5:
-      w3[2] = amd_bytealign_S (    0, w2[0], offset_minus_4);
-      w3[1] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
-      w3[0] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
-      w2[3] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
-      w2[2] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
-      w2[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w2[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w1[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w1[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w1[1] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w1[1] = w1[2];
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 6:
-      w3[2] = amd_bytealign_S (    0, w1[3], offset_minus_4);
-      w3[1] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
-      w3[0] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
-      w2[3] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
-      w2[2] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w2[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w2[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w1[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w1[2] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w1[2] = w1[3];
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 7:
-      w3[2] = amd_bytealign_S (    0, w1[2], offset_minus_4);
-      w3[1] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
-      w3[0] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
-      w2[3] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w2[2] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w2[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w2[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w1[3] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w1[3] = w2[0];
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 8:
-      w3[2] = amd_bytealign_S (    0, w1[1], offset_minus_4);
-      w3[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
-      w3[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w2[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w2[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w2[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w2[0] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w2[0] = w2[1];
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 9:
-      w3[2] = amd_bytealign_S (    0, w1[0], offset_minus_4);
-      w3[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
-      w3[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w2[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w2[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w2[1] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w2[1] = w2[2];
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 10:
-      w3[2] = amd_bytealign_S (    0, w0[3], offset_minus_4);
-      w3[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
-      w3[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w2[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w2[2] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w2[2] = w2[3];
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 11:
-      w3[2] = amd_bytealign_S (    0, w0[2], offset_minus_4);
-      w3[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
-      w3[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w2[3] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w2[3] = w3[0];
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 12:
-      w3[2] = amd_bytealign_S (    0, w0[1], offset_minus_4);
-      w3[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
-      w3[0] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w3[0] = w3[1];
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-
-    case 13:
-      w3[2] = amd_bytealign_S (    0, w0[0], offset_minus_4);
-      w3[1] = amd_bytealign_S (w0[0],     0, offset_minus_4);
-      w3[0] = 0;
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      if (offset_mod_4 == 0)
-      {
-        w3[1] = w3[2];
-        w3[2] = 0;
-      }
-
-      break;
-  }
-  #endif
-
-  #ifdef IS_NV
-  const int offset_minus_4 = 4 - (offset % 4);
-
-  const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff;
-
-  switch (offset / 4)
-  {
-    case 0:
-      w3[1] = __byte_perm_S (w3[0], w3[1], selector);
-      w3[0] = __byte_perm_S (w2[3], w3[0], selector);
-      w2[3] = __byte_perm_S (w2[2], w2[3], selector);
-      w2[2] = __byte_perm_S (w2[1], w2[2], selector);
-      w2[1] = __byte_perm_S (w2[0], w2[1], selector);
-      w2[0] = __byte_perm_S (w1[3], w2[0], selector);
-      w1[3] = __byte_perm_S (w1[2], w1[3], selector);
-      w1[2] = __byte_perm_S (w1[1], w1[2], selector);
-      w1[1] = __byte_perm_S (w1[0], w1[1], selector);
-      w1[0] = __byte_perm_S (w0[3], w1[0], selector);
-      w0[3] = __byte_perm_S (w0[2], w0[3], selector);
-      w0[2] = __byte_perm_S (w0[1], w0[2], selector);
-      w0[1] = __byte_perm_S (w0[0], w0[1], selector);
-      w0[0] = __byte_perm_S (    0, w0[0], selector);
-
-      break;
-
-    case 1:
-      w3[1] = __byte_perm_S (w2[3], w3[0], selector);
-      w3[0] = __byte_perm_S (w2[2], w2[3], selector);
-      w2[3] = __byte_perm_S (w2[1], w2[2], selector);
-      w2[2] = __byte_perm_S (w2[0], w2[1], selector);
-      w2[1] = __byte_perm_S (w1[3], w2[0], selector);
-      w2[0] = __byte_perm_S (w1[2], w1[3], selector);
-      w1[3] = __byte_perm_S (w1[1], w1[2], selector);
-      w1[2] = __byte_perm_S (w1[0], w1[1], selector);
-      w1[1] = __byte_perm_S (w0[3], w1[0], selector);
-      w1[0] = __byte_perm_S (w0[2], w0[3], selector);
-      w0[3] = __byte_perm_S (w0[1], w0[2], selector);
-      w0[2] = __byte_perm_S (w0[0], w0[1], selector);
-      w0[1] = __byte_perm_S (    0, w0[0], selector);
-      w0[0] = 0;
-
-      break;
-
-    case 2:
-      w3[1] = __byte_perm_S (w2[2], w2[3], selector);
-      w3[0] = __byte_perm_S (w2[1], w2[2], selector);
-      w2[3] = __byte_perm_S (w2[0], w2[1], selector);
-      w2[2] = __byte_perm_S (w1[3], w2[0], selector);
-      w2[1] = __byte_perm_S (w1[2], w1[3], selector);
-      w2[0] = __byte_perm_S (w1[1], w1[2], selector);
-      w1[3] = __byte_perm_S (w1[0], w1[1], selector);
-      w1[2] = __byte_perm_S (w0[3], w1[0], selector);
-      w1[1] = __byte_perm_S (w0[2], w0[3], selector);
-      w1[0] = __byte_perm_S (w0[1], w0[2], selector);
-      w0[3] = __byte_perm_S (w0[0], w0[1], selector);
-      w0[2] = __byte_perm_S (    0, w0[0], selector);
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 3:
-      w3[1] = __byte_perm_S (w2[1], w2[2], selector);
-      w3[0] = __byte_perm_S (w2[0], w2[1], selector);
-      w2[3] = __byte_perm_S (w1[3], w2[0], selector);
-      w2[2] = __byte_perm_S (w1[2], w1[3], selector);
-      w2[1] = __byte_perm_S (w1[1], w1[2], selector);
-      w2[0] = __byte_perm_S (w1[0], w1[1], selector);
-      w1[3] = __byte_perm_S (w0[3], w1[0], selector);
-      w1[2] = __byte_perm_S (w0[2], w0[3], selector);
-      w1[1] = __byte_perm_S (w0[1], w0[2], selector);
-      w1[0] = __byte_perm_S (w0[0], w0[1], selector);
-      w0[3] = __byte_perm_S (    0, w0[0], selector);
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 4:
-      w3[1] = __byte_perm_S (w2[0], w2[1], selector);
-      w3[0] = __byte_perm_S (w1[3], w2[0], selector);
-      w2[3] = __byte_perm_S (w1[2], w1[3], selector);
-      w2[2] = __byte_perm_S (w1[1], w1[2], selector);
-      w2[1] = __byte_perm_S (w1[0], w1[1], selector);
-      w2[0] = __byte_perm_S (w0[3], w1[0], selector);
-      w1[3] = __byte_perm_S (w0[2], w0[3], selector);
-      w1[2] = __byte_perm_S (w0[1], w0[2], selector);
-      w1[1] = __byte_perm_S (w0[0], w0[1], selector);
-      w1[0] = __byte_perm_S (    0, w0[0], selector);
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 5:
-      w3[1] = __byte_perm_S (w1[3], w2[0], selector);
-      w3[0] = __byte_perm_S (w1[2], w1[3], selector);
-      w2[3] = __byte_perm_S (w1[1], w1[2], selector);
-      w2[2] = __byte_perm_S (w1[0], w1[1], selector);
-      w2[1] = __byte_perm_S (w0[3], w1[0], selector);
-      w2[0] = __byte_perm_S (w0[2], w0[3], selector);
-      w1[3] = __byte_perm_S (w0[1], w0[2], selector);
-      w1[2] = __byte_perm_S (w0[0], w0[1], selector);
-      w1[1] = __byte_perm_S (    0, w0[0], selector);
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 6:
-      w3[1] = __byte_perm_S (w1[2], w1[3], selector);
-      w3[0] = __byte_perm_S (w1[1], w1[2], selector);
-      w2[3] = __byte_perm_S (w1[0], w1[1], selector);
-      w2[2] = __byte_perm_S (w0[3], w1[0], selector);
-      w2[1] = __byte_perm_S (w0[2], w0[3], selector);
-      w2[0] = __byte_perm_S (w0[1], w0[2], selector);
-      w1[3] = __byte_perm_S (w0[0], w0[1], selector);
-      w1[2] = __byte_perm_S (    0, w0[0], selector);
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 7:
-      w3[1] = __byte_perm_S (w1[1], w1[2], selector);
-      w3[0] = __byte_perm_S (w1[0], w1[1], selector);
-      w2[3] = __byte_perm_S (w0[3], w1[0], selector);
-      w2[2] = __byte_perm_S (w0[2], w0[3], selector);
-      w2[1] = __byte_perm_S (w0[1], w0[2], selector);
-      w2[0] = __byte_perm_S (w0[0], w0[1], selector);
-      w1[3] = __byte_perm_S (    0, w0[0], selector);
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 8:
-      w3[1] = __byte_perm_S (w1[0], w1[1], selector);
-      w3[0] = __byte_perm_S (w0[3], w1[0], selector);
-      w2[3] = __byte_perm_S (w0[2], w0[3], selector);
-      w2[2] = __byte_perm_S (w0[1], w0[2], selector);
-      w2[1] = __byte_perm_S (w0[0], w0[1], selector);
-      w2[0] = __byte_perm_S (    0, w0[0], selector);
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 9:
-      w3[1] = __byte_perm_S (w0[3], w1[0], selector);
-      w3[0] = __byte_perm_S (w0[2], w0[3], selector);
-      w2[3] = __byte_perm_S (w0[1], w0[2], selector);
-      w2[2] = __byte_perm_S (w0[0], w0[1], selector);
-      w2[1] = __byte_perm_S (    0, w0[0], selector);
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 10:
-      w3[1] = __byte_perm_S (w0[2], w0[3], selector);
-      w3[0] = __byte_perm_S (w0[1], w0[2], selector);
-      w2[3] = __byte_perm_S (w0[0], w0[1], selector);
-      w2[2] = __byte_perm_S (    0, w0[0], selector);
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 11:
-      w3[1] = __byte_perm_S (w0[1], w0[2], selector);
-      w3[0] = __byte_perm_S (w0[0], w0[1], selector);
-      w2[3] = __byte_perm_S (    0, w0[0], selector);
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 12:
-      w3[1] = __byte_perm_S (w0[0], w0[1], selector);
-      w3[0] = __byte_perm_S (    0, w0[0], selector);
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-
-    case 13:
-      w3[1] = __byte_perm_S (    0, w0[0], selector);
-      w3[0] = 0;
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-
-      break;
-  }
-  #endif
-}
-
-inline void switch_buffer_by_offset_be_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset)
-{
-  #if defined IS_AMD || defined IS_GENERIC
-  switch (offset / 4)
-  {
-    case 0:
-      w3[2] = amd_bytealign_S (w3[1],     0, offset);
-      w3[1] = amd_bytealign_S (w3[0], w3[1], offset);
-      w3[0] = amd_bytealign_S (w2[3], w3[0], offset);
-      w2[3] = amd_bytealign_S (w2[2], w2[3], offset);
-      w2[2] = amd_bytealign_S (w2[1], w2[2], offset);
-      w2[1] = amd_bytealign_S (w2[0], w2[1], offset);
-      w2[0] = amd_bytealign_S (w1[3], w2[0], offset);
-      w1[3] = amd_bytealign_S (w1[2], w1[3], offset);
-      w1[2] = amd_bytealign_S (w1[1], w1[2], offset);
-      w1[1] = amd_bytealign_S (w1[0], w1[1], offset);
-      w1[0] = amd_bytealign_S (w0[3], w1[0], offset);
-      w0[3] = amd_bytealign_S (w0[2], w0[3], offset);
-      w0[2] = amd_bytealign_S (w0[1], w0[2], offset);
-      w0[1] = amd_bytealign_S (w0[0], w0[1], offset);
-      w0[0] = amd_bytealign_S (    0, w0[0], offset);
-      break;
-
-    case 1:
-      w3[2] = amd_bytealign_S (w3[0],     0, offset);
-      w3[1] = amd_bytealign_S (w2[3], w3[0], offset);
-      w3[0] = amd_bytealign_S (w2[2], w2[3], offset);
-      w2[3] = amd_bytealign_S (w2[1], w2[2], offset);
-      w2[2] = amd_bytealign_S (w2[0], w2[1], offset);
-      w2[1] = amd_bytealign_S (w1[3], w2[0], offset);
-      w2[0] = amd_bytealign_S (w1[2], w1[3], offset);
-      w1[3] = amd_bytealign_S (w1[1], w1[2], offset);
-      w1[2] = amd_bytealign_S (w1[0], w1[1], offset);
-      w1[1] = amd_bytealign_S (w0[3], w1[0], offset);
-      w1[0] = amd_bytealign_S (w0[2], w0[3], offset);
-      w0[3] = amd_bytealign_S (w0[1], w0[2], offset);
-      w0[2] = amd_bytealign_S (w0[0], w0[1], offset);
-      w0[1] = amd_bytealign_S (    0, w0[0], offset);
-      w0[0] = 0;
-      break;
-
-    case 2:
-      w3[2] = amd_bytealign_S (w2[3],     0, offset);
-      w3[1] = amd_bytealign_S (w2[2], w2[3], offset);
-      w3[0] = amd_bytealign_S (w2[1], w2[2], offset);
-      w2[3] = amd_bytealign_S (w2[0], w2[1], offset);
-      w2[2] = amd_bytealign_S (w1[3], w2[0], offset);
-      w2[1] = amd_bytealign_S (w1[2], w1[3], offset);
-      w2[0] = amd_bytealign_S (w1[1], w1[2], offset);
-      w1[3] = amd_bytealign_S (w1[0], w1[1], offset);
-      w1[2] = amd_bytealign_S (w0[3], w1[0], offset);
-      w1[1] = amd_bytealign_S (w0[2], w0[3], offset);
-      w1[0] = amd_bytealign_S (w0[1], w0[2], offset);
-      w0[3] = amd_bytealign_S (w0[0], w0[1], offset);
-      w0[2] = amd_bytealign_S (    0, w0[0], offset);
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 3:
-      w3[2] = amd_bytealign_S (w2[2],     0, offset);
-      w3[1] = amd_bytealign_S (w2[1], w2[2], offset);
-      w3[0] = amd_bytealign_S (w2[0], w2[1], offset);
-      w2[3] = amd_bytealign_S (w1[3], w2[0], offset);
-      w2[2] = amd_bytealign_S (w1[2], w1[3], offset);
-      w2[1] = amd_bytealign_S (w1[1], w1[2], offset);
-      w2[0] = amd_bytealign_S (w1[0], w1[1], offset);
-      w1[3] = amd_bytealign_S (w0[3], w1[0], offset);
-      w1[2] = amd_bytealign_S (w0[2], w0[3], offset);
-      w1[1] = amd_bytealign_S (w0[1], w0[2], offset);
-      w1[0] = amd_bytealign_S (w0[0], w0[1], offset);
-      w0[3] = amd_bytealign_S (    0, w0[0], offset);
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 4:
-      w3[2] = amd_bytealign_S (w2[1],     0, offset);
-      w3[1] = amd_bytealign_S (w2[0], w2[1], offset);
-      w3[0] = amd_bytealign_S (w1[3], w2[0], offset);
-      w2[3] = amd_bytealign_S (w1[2], w1[3], offset);
-      w2[2] = amd_bytealign_S (w1[1], w1[2], offset);
-      w2[1] = amd_bytealign_S (w1[0], w1[1], offset);
-      w2[0] = amd_bytealign_S (w0[3], w1[0], offset);
-      w1[3] = amd_bytealign_S (w0[2], w0[3], offset);
-      w1[2] = amd_bytealign_S (w0[1], w0[2], offset);
-      w1[1] = amd_bytealign_S (w0[0], w0[1], offset);
-      w1[0] = amd_bytealign_S (    0, w0[0], offset);
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 5:
-      w3[2] = amd_bytealign_S (w2[0],     0, offset);
-      w3[1] = amd_bytealign_S (w1[3], w2[0], offset);
-      w3[0] = amd_bytealign_S (w1[2], w1[3], offset);
-      w2[3] = amd_bytealign_S (w1[1], w1[2], offset);
-      w2[2] = amd_bytealign_S (w1[0], w1[1], offset);
-      w2[1] = amd_bytealign_S (w0[3], w1[0], offset);
-      w2[0] = amd_bytealign_S (w0[2], w0[3], offset);
-      w1[3] = amd_bytealign_S (w0[1], w0[2], offset);
-      w1[2] = amd_bytealign_S (w0[0], w0[1], offset);
-      w1[1] = amd_bytealign_S (    0, w0[0], offset);
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 6:
-      w3[2] = amd_bytealign_S (w1[3],     0, offset);
-      w3[1] = amd_bytealign_S (w1[2], w1[3], offset);
-      w3[0] = amd_bytealign_S (w1[1], w1[2], offset);
-      w2[3] = amd_bytealign_S (w1[0], w1[1], offset);
-      w2[2] = amd_bytealign_S (w0[3], w1[0], offset);
-      w2[1] = amd_bytealign_S (w0[2], w0[3], offset);
-      w2[0] = amd_bytealign_S (w0[1], w0[2], offset);
-      w1[3] = amd_bytealign_S (w0[0], w0[1], offset);
-      w1[2] = amd_bytealign_S (    0, w0[0], offset);
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 7:
-      w3[2] = amd_bytealign_S (w1[2],     0, offset);
-      w3[1] = amd_bytealign_S (w1[1], w1[2], offset);
-      w3[0] = amd_bytealign_S (w1[0], w1[1], offset);
-      w2[3] = amd_bytealign_S (w0[3], w1[0], offset);
-      w2[2] = amd_bytealign_S (w0[2], w0[3], offset);
-      w2[1] = amd_bytealign_S (w0[1], w0[2], offset);
-      w2[0] = amd_bytealign_S (w0[0], w0[1], offset);
-      w1[3] = amd_bytealign_S (    0, w0[0], offset);
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 8:
-      w3[2] = amd_bytealign_S (w1[1],     0, offset);
-      w3[1] = amd_bytealign_S (w1[0], w1[1], offset);
-      w3[0] = amd_bytealign_S (w0[3], w1[0], offset);
-      w2[3] = amd_bytealign_S (w0[2], w0[3], offset);
-      w2[2] = amd_bytealign_S (w0[1], w0[2], offset);
-      w2[1] = amd_bytealign_S (w0[0], w0[1], offset);
-      w2[0] = amd_bytealign_S (    0, w0[0], offset);
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 9:
-      w3[2] = amd_bytealign_S (w1[0],     0, offset);
-      w3[1] = amd_bytealign_S (w0[3], w1[0], offset);
-      w3[0] = amd_bytealign_S (w0[2], w0[3], offset);
-      w2[3] = amd_bytealign_S (w0[1], w0[2], offset);
-      w2[2] = amd_bytealign_S (w0[0], w0[1], offset);
-      w2[1] = amd_bytealign_S (    0, w0[0], offset);
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 10:
-      w3[2] = amd_bytealign_S (w0[3],     0, offset);
-      w3[1] = amd_bytealign_S (w0[2], w0[3], offset);
-      w3[0] = amd_bytealign_S (w0[1], w0[2], offset);
-      w2[3] = amd_bytealign_S (w0[0], w0[1], offset);
-      w2[2] = amd_bytealign_S (    0, w0[0], offset);
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 11:
-      w3[2] = amd_bytealign_S (w0[2],     0, offset);
-      w3[1] = amd_bytealign_S (w0[1], w0[2], offset);
-      w3[0] = amd_bytealign_S (w0[0], w0[1], offset);
-      w2[3] = amd_bytealign_S (    0, w0[0], offset);
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 12:
-      w3[2] = amd_bytealign_S (w0[1],     0, offset);
-      w3[1] = amd_bytealign_S (w0[0], w0[1], offset);
-      w3[0] = amd_bytealign_S (    0, w0[0], offset);
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 13:
-      w3[2] = amd_bytealign_S (w0[0],     0, offset);
-      w3[1] = amd_bytealign_S (    0, w0[0], offset);
-      w3[0] = 0;
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-  }
-  #endif
-
-  #ifdef IS_NV
-  const int selector = (0x76543210 >> ((offset & 3) * 4)) & 0xffff;
-
-  switch (offset / 4)
-  {
-    case 0:
-      w3[1] = __byte_perm_S (w3[1], w3[0], selector);
-      w3[0] = __byte_perm_S (w3[0], w2[3], selector);
-      w2[3] = __byte_perm_S (w2[3], w2[2], selector);
-      w2[2] = __byte_perm_S (w2[2], w2[1], selector);
-      w2[1] = __byte_perm_S (w2[1], w2[0], selector);
-      w2[0] = __byte_perm_S (w2[0], w1[3], selector);
-      w1[3] = __byte_perm_S (w1[3], w1[2], selector);
-      w1[2] = __byte_perm_S (w1[2], w1[1], selector);
-      w1[1] = __byte_perm_S (w1[1], w1[0], selector);
-      w1[0] = __byte_perm_S (w1[0], w0[3], selector);
-      w0[3] = __byte_perm_S (w0[3], w0[2], selector);
-      w0[2] = __byte_perm_S (w0[2], w0[1], selector);
-      w0[1] = __byte_perm_S (w0[1], w0[0], selector);
-      w0[0] = __byte_perm_S (w0[0],     0, selector);
-      break;
-
-    case 1:
-      w3[1] = __byte_perm_S (w3[0], w2[3], selector);
-      w3[0] = __byte_perm_S (w2[3], w2[2], selector);
-      w2[3] = __byte_perm_S (w2[2], w2[1], selector);
-      w2[2] = __byte_perm_S (w2[1], w2[0], selector);
-      w2[1] = __byte_perm_S (w2[0], w1[3], selector);
-      w2[0] = __byte_perm_S (w1[3], w1[2], selector);
-      w1[3] = __byte_perm_S (w1[2], w1[1], selector);
-      w1[2] = __byte_perm_S (w1[1], w1[0], selector);
-      w1[1] = __byte_perm_S (w1[0], w0[3], selector);
-      w1[0] = __byte_perm_S (w0[3], w0[2], selector);
-      w0[3] = __byte_perm_S (w0[2], w0[1], selector);
-      w0[2] = __byte_perm_S (w0[1], w0[0], selector);
-      w0[1] = __byte_perm_S (w0[0],     0, selector);
-      w0[0] = 0;
-      break;
-
-    case 2:
-      w3[1] = __byte_perm_S (w2[3], w2[2], selector);
-      w3[0] = __byte_perm_S (w2[2], w2[1], selector);
-      w2[3] = __byte_perm_S (w2[1], w2[0], selector);
-      w2[2] = __byte_perm_S (w2[0], w1[3], selector);
-      w2[1] = __byte_perm_S (w1[3], w1[2], selector);
-      w2[0] = __byte_perm_S (w1[2], w1[1], selector);
-      w1[3] = __byte_perm_S (w1[1], w1[0], selector);
-      w1[2] = __byte_perm_S (w1[0], w0[3], selector);
-      w1[1] = __byte_perm_S (w0[3], w0[2], selector);
-      w1[0] = __byte_perm_S (w0[2], w0[1], selector);
-      w0[3] = __byte_perm_S (w0[1], w0[0], selector);
-      w0[2] = __byte_perm_S (w0[0],     0, selector);
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 3:
-      w3[1] = __byte_perm_S (w2[2], w2[1], selector);
-      w3[0] = __byte_perm_S (w2[1], w2[0], selector);
-      w2[3] = __byte_perm_S (w2[0], w1[3], selector);
-      w2[2] = __byte_perm_S (w1[3], w1[2], selector);
-      w2[1] = __byte_perm_S (w1[2], w1[1], selector);
-      w2[0] = __byte_perm_S (w1[1], w1[0], selector);
-      w1[3] = __byte_perm_S (w1[0], w0[3], selector);
-      w1[2] = __byte_perm_S (w0[3], w0[2], selector);
-      w1[1] = __byte_perm_S (w0[2], w0[1], selector);
-      w1[0] = __byte_perm_S (w0[1], w0[0], selector);
-      w0[3] = __byte_perm_S (w0[0],     0, selector);
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 4:
-      w3[1] = __byte_perm_S (w2[1], w2[0], selector);
-      w3[0] = __byte_perm_S (w2[0], w1[3], selector);
-      w2[3] = __byte_perm_S (w1[3], w1[2], selector);
-      w2[2] = __byte_perm_S (w1[2], w1[1], selector);
-      w2[1] = __byte_perm_S (w1[1], w1[0], selector);
-      w2[0] = __byte_perm_S (w1[0], w0[3], selector);
-      w1[3] = __byte_perm_S (w0[3], w0[2], selector);
-      w1[2] = __byte_perm_S (w0[2], w0[1], selector);
-      w1[1] = __byte_perm_S (w0[1], w0[0], selector);
-      w1[0] = __byte_perm_S (w0[0],     0, selector);
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 5:
-      w3[1] = __byte_perm_S (w2[0], w1[3], selector);
-      w3[0] = __byte_perm_S (w1[3], w1[2], selector);
-      w2[3] = __byte_perm_S (w1[2], w1[1], selector);
-      w2[2] = __byte_perm_S (w1[1], w1[0], selector);
-      w2[1] = __byte_perm_S (w1[0], w0[3], selector);
-      w2[0] = __byte_perm_S (w0[3], w0[2], selector);
-      w1[3] = __byte_perm_S (w0[2], w0[1], selector);
-      w1[2] = __byte_perm_S (w0[1], w0[0], selector);
-      w1[1] = __byte_perm_S (w0[0],     0, selector);
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 6:
-      w3[1] = __byte_perm_S (w1[3], w1[2], selector);
-      w3[0] = __byte_perm_S (w1[2], w1[1], selector);
-      w2[3] = __byte_perm_S (w1[1], w1[0], selector);
-      w2[2] = __byte_perm_S (w1[0], w0[3], selector);
-      w2[1] = __byte_perm_S (w0[3], w0[2], selector);
-      w2[0] = __byte_perm_S (w0[2], w0[1], selector);
-      w1[3] = __byte_perm_S (w0[1], w0[0], selector);
-      w1[2] = __byte_perm_S (w0[0],     0, selector);
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 7:
-      w3[1] = __byte_perm_S (w1[2], w1[1], selector);
-      w3[0] = __byte_perm_S (w1[1], w1[0], selector);
-      w2[3] = __byte_perm_S (w1[0], w0[3], selector);
-      w2[2] = __byte_perm_S (w0[3], w0[2], selector);
-      w2[1] = __byte_perm_S (w0[2], w0[1], selector);
-      w2[0] = __byte_perm_S (w0[1], w0[0], selector);
-      w1[3] = __byte_perm_S (w0[0],     0, selector);
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 8:
-      w3[1] = __byte_perm_S (w1[1], w1[0], selector);
-      w3[0] = __byte_perm_S (w1[0], w0[3], selector);
-      w2[3] = __byte_perm_S (w0[3], w0[2], selector);
-      w2[2] = __byte_perm_S (w0[2], w0[1], selector);
-      w2[1] = __byte_perm_S (w0[1], w0[0], selector);
-      w2[0] = __byte_perm_S (w0[0],     0, selector);
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 9:
-      w3[1] = __byte_perm_S (w1[0], w0[3], selector);
-      w3[0] = __byte_perm_S (w0[3], w0[2], selector);
-      w2[3] = __byte_perm_S (w0[2], w0[1], selector);
-      w2[2] = __byte_perm_S (w0[1], w0[0], selector);
-      w2[1] = __byte_perm_S (w0[0],     0, selector);
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 10:
-      w3[1] = __byte_perm_S (w0[3], w0[2], selector);
-      w3[0] = __byte_perm_S (w0[2], w0[1], selector);
-      w2[3] = __byte_perm_S (w0[1], w0[0], selector);
-      w2[2] = __byte_perm_S (w0[0],     0, selector);
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 11:
-      w3[1] = __byte_perm_S (w0[2], w0[1], selector);
-      w3[0] = __byte_perm_S (w0[1], w0[0], selector);
-      w2[3] = __byte_perm_S (w0[0],     0, selector);
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 12:
-      w3[1] = __byte_perm_S (w0[1], w0[0], selector);
-      w3[0] = __byte_perm_S (w0[0],     0, selector);
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-
-    case 13:
-      w3[1] = __byte_perm_S (w0[0],     0, selector);
-      w3[0] = 0;
-      w2[3] = 0;
-      w2[2] = 0;
-      w2[1] = 0;
-      w2[0] = 0;
-      w1[3] = 0;
-      w1[2] = 0;
-      w1[1] = 0;
-      w1[0] = 0;
-      w0[3] = 0;
-      w0[2] = 0;
-      w0[1] = 0;
-      w0[0] = 0;
-      break;
-  }
-  #endif
-}
-
-/**
- * vector functions on scalar types (for inner loop usage)
- */
-
-#define PACKVS2(sn,vn,e)  \
-  sn[0] = vn[0].s##e;     \
-  sn[1] = vn[1].s##e;
-
-#define PACKSV2(sn,vn,e)  \
-  vn[0].s##e = sn[0];     \
-  vn[1].s##e = sn[1];
-
-#define PACKVS24(s0,s1,v0,v1,e) \
-  PACKVS4 (s0, v0, e);          \
-  PACKVS4 (s1, v1, e);
-
-#define PACKSV24(s0,s1,v0,v1,e) \
-  PACKSV4 (s0, v0, e);          \
-  PACKSV4 (s1, v1, e);
-
-#define PACKVS4(sn,vn,e)  \
-  sn[0] = vn[0].s##e;     \
-  sn[1] = vn[1].s##e;     \
-  sn[2] = vn[2].s##e;     \
-  sn[3] = vn[3].s##e;
-
-#define PACKSV4(sn,vn,e)  \
-  vn[0].s##e = sn[0];     \
-  vn[1].s##e = sn[1];     \
-  vn[2].s##e = sn[2];     \
-  vn[3].s##e = sn[3];
-
-#define PACKVS44(s0,s1,s2,s3,v0,v1,v2,v3,e) \
-  PACKVS4 (s0, v0, e);                      \
-  PACKVS4 (s1, v1, e);                      \
-  PACKVS4 (s2, v2, e);                      \
-  PACKVS4 (s3, v3, e);
-
-#define PACKSV44(s0,s1,s2,s3,v0,v1,v2,v3,e) \
-  PACKSV4 (s0, v0, e);                      \
-  PACKSV4 (s1, v1, e);                      \
-  PACKSV4 (s2, v2, e);                      \
-  PACKSV4 (s3, v3, e);
-
-inline void switch_buffer_by_offset_le_VV (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x offset)
-{
-  #if VECT_SIZE == 1
-
-  switch_buffer_by_offset_le_S (w0, w1, w2, w3, offset);
-
-  #else
-
-  u32 t0[4];
-  u32 t1[4];
-  u32 t2[4];
-  u32 t3[4];
-
-  #endif
-
-  #if   VECT_SIZE == 2
-
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
-
-  #elif VECT_SIZE == 4
-
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
-
-  #elif VECT_SIZE == 8
-
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7);
-
-  #elif VECT_SIZE == 16
-
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s8); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 8);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s9); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 9);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, a); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sa); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, a);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, b); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sb); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, b);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, c); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sc); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, c);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, d); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sd); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, d);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, e); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.se); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, e);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, f); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sf); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, f);
-
-  #endif
-}
-
-inline void append_0x01_2x4_VV (u32x w0[4], u32x w1[4], const u32x offset)
-{
-  #if VECT_SIZE == 1
-
-  append_0x01_2x4_S (w0, w1, offset);
-
-  #else
-
-  u32 t0[4];
-  u32 t1[4];
-
-  #endif
-
-  #if   VECT_SIZE == 2
-
-  PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
-  PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
-
-  #elif VECT_SIZE == 4
-
-  PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
-  PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
-  PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
-  PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
-
-  #elif VECT_SIZE == 8
-
-  PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
-  PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
-  PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
-  PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
-  PACKVS24 (t0, t1, w0, w1, 4); append_0x01_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4);
-  PACKVS24 (t0, t1, w0, w1, 5); append_0x01_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5);
-  PACKVS24 (t0, t1, w0, w1, 6); append_0x01_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6);
-  PACKVS24 (t0, t1, w0, w1, 7); append_0x01_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7);
-
-  #elif VECT_SIZE == 16
-
-  PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
-  PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
-  PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
-  PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
-  PACKVS24 (t0, t1, w0, w1, 4); append_0x01_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4);
-  PACKVS24 (t0, t1, w0, w1, 5); append_0x01_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5);
-  PACKVS24 (t0, t1, w0, w1, 6); append_0x01_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6);
-  PACKVS24 (t0, t1, w0, w1, 7); append_0x01_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7);
-  PACKVS24 (t0, t1, w0, w1, 8); append_0x01_2x4_S (t0, t1, offset.s8); PACKSV24 (t0, t1, w0, w1, 8);
-  PACKVS24 (t0, t1, w0, w1, 9); append_0x01_2x4_S (t0, t1, offset.s9); PACKSV24 (t0, t1, w0, w1, 9);
-  PACKVS24 (t0, t1, w0, w1, a); append_0x01_2x4_S (t0, t1, offset.sa); PACKSV24 (t0, t1, w0, w1, a);
-  PACKVS24 (t0, t1, w0, w1, b); append_0x01_2x4_S (t0, t1, offset.sb); PACKSV24 (t0, t1, w0, w1, b);
-  PACKVS24 (t0, t1, w0, w1, c); append_0x01_2x4_S (t0, t1, offset.sc); PACKSV24 (t0, t1, w0, w1, c);
-  PACKVS24 (t0, t1, w0, w1, d); append_0x01_2x4_S (t0, t1, offset.sd); PACKSV24 (t0, t1, w0, w1, d);
-  PACKVS24 (t0, t1, w0, w1, e); append_0x01_2x4_S (t0, t1, offset.se); PACKSV24 (t0, t1, w0, w1, e);
-  PACKVS24 (t0, t1, w0, w1, f); append_0x01_2x4_S (t0, t1, offset.sf); PACKSV24 (t0, t1, w0, w1, f);
-
-  #endif
-}
-
-inline void append_0x80_2x4_VV (u32x w0[4], u32x w1[4], const u32x offset)
-{
-  #if VECT_SIZE == 1
-
-  append_0x80_2x4_S (w0, w1, offset);
-
-  #else
-
-  u32 t0[4];
-  u32 t1[4];
-
-  #endif
-
-  #if   VECT_SIZE == 2
-
-  PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
-  PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
-
-  #elif VECT_SIZE == 4
-
-  PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
-  PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
-  PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
-  PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
-
-  #elif VECT_SIZE == 8
-
-  PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
-  PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
-  PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
-  PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
-  PACKVS24 (t0, t1, w0, w1, 4); append_0x80_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4);
-  PACKVS24 (t0, t1, w0, w1, 5); append_0x80_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5);
-  PACKVS24 (t0, t1, w0, w1, 6); append_0x80_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6);
-  PACKVS24 (t0, t1, w0, w1, 7); append_0x80_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7);
-
-  #elif VECT_SIZE == 16
-
-  PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
-  PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
-  PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
-  PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
-  PACKVS24 (t0, t1, w0, w1, 4); append_0x80_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4);
-  PACKVS24 (t0, t1, w0, w1, 5); append_0x80_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5);
-  PACKVS24 (t0, t1, w0, w1, 6); append_0x80_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6);
-  PACKVS24 (t0, t1, w0, w1, 7); append_0x80_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7);
-  PACKVS24 (t0, t1, w0, w1, 8); append_0x80_2x4_S (t0, t1, offset.s8); PACKSV24 (t0, t1, w0, w1, 8);
-  PACKVS24 (t0, t1, w0, w1, 9); append_0x80_2x4_S (t0, t1, offset.s9); PACKSV24 (t0, t1, w0, w1, 9);
-  PACKVS24 (t0, t1, w0, w1, a); append_0x80_2x4_S (t0, t1, offset.sa); PACKSV24 (t0, t1, w0, w1, a);
-  PACKVS24 (t0, t1, w0, w1, b); append_0x80_2x4_S (t0, t1, offset.sb); PACKSV24 (t0, t1, w0, w1, b);
-  PACKVS24 (t0, t1, w0, w1, c); append_0x80_2x4_S (t0, t1, offset.sc); PACKSV24 (t0, t1, w0, w1, c);
-  PACKVS24 (t0, t1, w0, w1, d); append_0x80_2x4_S (t0, t1, offset.sd); PACKSV24 (t0, t1, w0, w1, d);
-  PACKVS24 (t0, t1, w0, w1, e); append_0x80_2x4_S (t0, t1, offset.se); PACKSV24 (t0, t1, w0, w1, e);
-  PACKVS24 (t0, t1, w0, w1, f); append_0x80_2x4_S (t0, t1, offset.sf); PACKSV24 (t0, t1, w0, w1, f);
-
-  #endif
-}
-
-inline void append_0x80_4x4_VV (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x offset)
-{
-  #if VECT_SIZE == 1
-
-  append_0x80_4x4_S (w0, w1, w2, w3, offset);
-
-  #else
-
-  u32 t0[4];
-  u32 t1[4];
-  u32 t2[4];
-  u32 t3[4];
-
-  #endif
-
-  #if   VECT_SIZE == 2
-
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
-
-  #elif VECT_SIZE == 4
-
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
-
-  #elif VECT_SIZE == 8
-
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); append_0x80_4x4_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); append_0x80_4x4_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); append_0x80_4x4_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); append_0x80_4x4_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7);
-
-  #elif VECT_SIZE == 16
-
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); append_0x80_4x4_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); append_0x80_4x4_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); append_0x80_4x4_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); append_0x80_4x4_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); append_0x80_4x4_S (t0, t1, t2, t3, offset.s8); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 8);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); append_0x80_4x4_S (t0, t1, t2, t3, offset.s9); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 9);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, a); append_0x80_4x4_S (t0, t1, t2, t3, offset.sa); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, a);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, b); append_0x80_4x4_S (t0, t1, t2, t3, offset.sb); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, b);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, c); append_0x80_4x4_S (t0, t1, t2, t3, offset.sc); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, c);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, d); append_0x80_4x4_S (t0, t1, t2, t3, offset.sd); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, d);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, e); append_0x80_4x4_S (t0, t1, t2, t3, offset.se); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, e);
-  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, f); append_0x80_4x4_S (t0, t1, t2, t3, offset.sf); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, f);
-
-  #endif
-}

diff --git a/OpenCL/inc_cipher_aes256.cl b/OpenCL/inc_cipher_aes256.cl
new file mode 100644 (file)
index 0000000..b3b6107
--- /dev/null
+++ b/OpenCL/inc_cipher_aes256.cl
@@ -0,0 +1,1048 @@
+/**
+ * Author......: Jens Steube <jens.steube@gmail.com>
+ * License.....: MIT
+ */
+
+__constant u32 te0[256] =
+{
+  0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d,
+  0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554,
+  0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d,
+  0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a,
+  0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87,
+  0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b,
+  0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea,
+  0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b,
+  0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a,
+  0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f,
+  0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108,
+  0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f,
+  0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e,
+  0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5,
+  0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d,
+  0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f,
+  0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e,
+  0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb,
+  0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce,
+  0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497,
+  0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c,
+  0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed,
+  0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b,
+  0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a,
+  0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16,
+  0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594,
+  0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81,
+  0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3,
+  0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a,
+  0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504,
+  0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163,
+  0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d,
+  0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f,
+  0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739,
+  0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47,
+  0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395,
+  0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f,
+  0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883,
+  0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c,
+  0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76,
+  0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e,
+  0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4,
+  0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6,
+  0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b,
+  0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7,
+  0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0,
+  0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25,
+  0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818,
+  0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72,
+  0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651,
+  0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21,
+  0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85,
+  0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa,
+  0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12,
+  0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0,
+  0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9,
+  0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133,
+  0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7,
+  0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920,
+  0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a,
+  0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17,
+  0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8,
+  0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11,
+  0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a,
+};
+
+__constant u32 te1[256] =
+{
+  0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b,
+  0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5,
+  0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b,
+  0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676,
+  0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d,
+  0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0,
+  0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf,
+  0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0,
+  0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626,
+  0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc,
+  0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1,
+  0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515,
+  0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3,
+  0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a,
+  0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2,
+  0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575,
+  0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a,
+  0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0,
+  0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3,
+  0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484,
+  0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded,
+  0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b,
+  0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939,
+  0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf,
+  0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb,
+  0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585,
+  0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f,
+  0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8,
+  0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f,
+  0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5,
+  0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121,
+  0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2,
+  0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec,
+  0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717,
+  0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d,
+  0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373,
+  0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc,
+  0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888,
+  0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414,
+  0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb,
+  0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a,
+  0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c,
+  0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262,
+  0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979,
+  0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d,
+  0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9,
+  0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea,
+  0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808,
+  0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e,
+  0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6,
+  0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f,
+  0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a,
+  0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666,
+  0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e,
+  0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9,
+  0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e,
+  0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111,
+  0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494,
+  0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9,
+  0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf,
+  0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d,
+  0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868,
+  0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f,
+  0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616,
+};
+
+__constant u32 te2[256] =
+{
+  0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b,
+  0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5,
+  0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b,
+  0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76,
+  0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d,
+  0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0,
+  0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af,
+  0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0,
+  0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26,
+  0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc,
+  0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1,
+  0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15,
+  0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3,
+  0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a,
+  0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2,
+  0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75,
+  0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a,
+  0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0,
+  0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3,
+  0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384,
+  0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed,
+  0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b,
+  0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239,
+  0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf,
+  0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb,
+  0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185,
+  0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f,
+  0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8,
+  0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f,
+  0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5,
+  0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221,
+  0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2,
+  0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec,
+  0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17,
+  0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d,
+  0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673,
+  0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc,
+  0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88,
+  0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814,
+  0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb,
+  0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a,
+  0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c,
+  0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462,
+  0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279,
+  0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d,
+  0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9,
+  0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea,
+  0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008,
+  0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e,
+  0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6,
+  0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f,
+  0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a,
+  0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66,
+  0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e,
+  0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9,
+  0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e,
+  0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211,
+  0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394,
+  0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9,
+  0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df,
+  0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d,
+  0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068,
+  0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f,
+  0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16,
+};
+
+__constant u32 te3[256] =
+{
+  0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6,
+  0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491,
+  0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56,
+  0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec,
+  0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa,
+  0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb,
+  0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45,
+  0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b,
+  0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c,
+  0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83,
+  0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9,
+  0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a,
+  0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d,
+  0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f,
+  0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf,
+  0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea,
+  0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34,
+  0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b,
+  0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d,
+  0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713,
+  0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1,
+  0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6,
+  0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72,
+  0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85,
+  0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed,
+  0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411,
+  0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe,
+  0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b,
+  0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05,
+  0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1,
+  0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342,
+  0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf,
+  0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3,
+  0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e,
+  0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a,
+  0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6,
+  0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3,
+  0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b,
+  0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28,
+  0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad,
+  0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14,
+  0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8,
+  0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4,
+  0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2,
+  0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da,
+  0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049,
+  0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf,
+  0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810,
+  0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c,
+  0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197,
+  0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e,
+  0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f,
+  0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc,
+  0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c,
+  0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069,
+  0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927,
+  0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322,
+  0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733,
+  0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9,
+  0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5,
+  0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a,
+  0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0,
+  0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e,
+  0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c,
+};
+
+__constant u32 te4[256] =
+{
+  0x63636363, 0x7c7c7c7c, 0x77777777, 0x7b7b7b7b,
+  0xf2f2f2f2, 0x6b6b6b6b, 0x6f6f6f6f, 0xc5c5c5c5,
+  0x30303030, 0x01010101, 0x67676767, 0x2b2b2b2b,
+  0xfefefefe, 0xd7d7d7d7, 0xabababab, 0x76767676,
+  0xcacacaca, 0x82828282, 0xc9c9c9c9, 0x7d7d7d7d,
+  0xfafafafa, 0x59595959, 0x47474747, 0xf0f0f0f0,
+  0xadadadad, 0xd4d4d4d4, 0xa2a2a2a2, 0xafafafaf,
+  0x9c9c9c9c, 0xa4a4a4a4, 0x72727272, 0xc0c0c0c0,
+  0xb7b7b7b7, 0xfdfdfdfd, 0x93939393, 0x26262626,
+  0x36363636, 0x3f3f3f3f, 0xf7f7f7f7, 0xcccccccc,
+  0x34343434, 0xa5a5a5a5, 0xe5e5e5e5, 0xf1f1f1f1,
+  0x71717171, 0xd8d8d8d8, 0x31313131, 0x15151515,
+  0x04040404, 0xc7c7c7c7, 0x23232323, 0xc3c3c3c3,
+  0x18181818, 0x96969696, 0x05050505, 0x9a9a9a9a,
+  0x07070707, 0x12121212, 0x80808080, 0xe2e2e2e2,
+  0xebebebeb, 0x27272727, 0xb2b2b2b2, 0x75757575,
+  0x09090909, 0x83838383, 0x2c2c2c2c, 0x1a1a1a1a,
+  0x1b1b1b1b, 0x6e6e6e6e, 0x5a5a5a5a, 0xa0a0a0a0,
+  0x52525252, 0x3b3b3b3b, 0xd6d6d6d6, 0xb3b3b3b3,
+  0x29292929, 0xe3e3e3e3, 0x2f2f2f2f, 0x84848484,
+  0x53535353, 0xd1d1d1d1, 0x00000000, 0xedededed,
+  0x20202020, 0xfcfcfcfc, 0xb1b1b1b1, 0x5b5b5b5b,
+  0x6a6a6a6a, 0xcbcbcbcb, 0xbebebebe, 0x39393939,
+  0x4a4a4a4a, 0x4c4c4c4c, 0x58585858, 0xcfcfcfcf,
+  0xd0d0d0d0, 0xefefefef, 0xaaaaaaaa, 0xfbfbfbfb,
+  0x43434343, 0x4d4d4d4d, 0x33333333, 0x85858585,
+  0x45454545, 0xf9f9f9f9, 0x02020202, 0x7f7f7f7f,
+  0x50505050, 0x3c3c3c3c, 0x9f9f9f9f, 0xa8a8a8a8,
+  0x51515151, 0xa3a3a3a3, 0x40404040, 0x8f8f8f8f,
+  0x92929292, 0x9d9d9d9d, 0x38383838, 0xf5f5f5f5,
+  0xbcbcbcbc, 0xb6b6b6b6, 0xdadadada, 0x21212121,
+  0x10101010, 0xffffffff, 0xf3f3f3f3, 0xd2d2d2d2,
+  0xcdcdcdcd, 0x0c0c0c0c, 0x13131313, 0xecececec,
+  0x5f5f5f5f, 0x97979797, 0x44444444, 0x17171717,
+  0xc4c4c4c4, 0xa7a7a7a7, 0x7e7e7e7e, 0x3d3d3d3d,
+  0x64646464, 0x5d5d5d5d, 0x19191919, 0x73737373,
+  0x60606060, 0x81818181, 0x4f4f4f4f, 0xdcdcdcdc,
+  0x22222222, 0x2a2a2a2a, 0x90909090, 0x88888888,
+  0x46464646, 0xeeeeeeee, 0xb8b8b8b8, 0x14141414,
+  0xdededede, 0x5e5e5e5e, 0x0b0b0b0b, 0xdbdbdbdb,
+  0xe0e0e0e0, 0x32323232, 0x3a3a3a3a, 0x0a0a0a0a,
+  0x49494949, 0x06060606, 0x24242424, 0x5c5c5c5c,
+  0xc2c2c2c2, 0xd3d3d3d3, 0xacacacac, 0x62626262,
+  0x91919191, 0x95959595, 0xe4e4e4e4, 0x79797979,
+  0xe7e7e7e7, 0xc8c8c8c8, 0x37373737, 0x6d6d6d6d,
+  0x8d8d8d8d, 0xd5d5d5d5, 0x4e4e4e4e, 0xa9a9a9a9,
+  0x6c6c6c6c, 0x56565656, 0xf4f4f4f4, 0xeaeaeaea,
+  0x65656565, 0x7a7a7a7a, 0xaeaeaeae, 0x08080808,
+  0xbabababa, 0x78787878, 0x25252525, 0x2e2e2e2e,
+  0x1c1c1c1c, 0xa6a6a6a6, 0xb4b4b4b4, 0xc6c6c6c6,
+  0xe8e8e8e8, 0xdddddddd, 0x74747474, 0x1f1f1f1f,
+  0x4b4b4b4b, 0xbdbdbdbd, 0x8b8b8b8b, 0x8a8a8a8a,
+  0x70707070, 0x3e3e3e3e, 0xb5b5b5b5, 0x66666666,
+  0x48484848, 0x03030303, 0xf6f6f6f6, 0x0e0e0e0e,
+  0x61616161, 0x35353535, 0x57575757, 0xb9b9b9b9,
+  0x86868686, 0xc1c1c1c1, 0x1d1d1d1d, 0x9e9e9e9e,
+  0xe1e1e1e1, 0xf8f8f8f8, 0x98989898, 0x11111111,
+  0x69696969, 0xd9d9d9d9, 0x8e8e8e8e, 0x94949494,
+  0x9b9b9b9b, 0x1e1e1e1e, 0x87878787, 0xe9e9e9e9,
+  0xcececece, 0x55555555, 0x28282828, 0xdfdfdfdf,
+  0x8c8c8c8c, 0xa1a1a1a1, 0x89898989, 0x0d0d0d0d,
+  0xbfbfbfbf, 0xe6e6e6e6, 0x42424242, 0x68686868,
+  0x41414141, 0x99999999, 0x2d2d2d2d, 0x0f0f0f0f,
+  0xb0b0b0b0, 0x54545454, 0xbbbbbbbb, 0x16161616,
+};
+
+__constant u32 td0[256] =
+{
+  0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96,
+  0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393,
+  0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25,
+  0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f,
+  0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1,
+  0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6,
+  0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da,
+  0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844,
+  0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd,
+  0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4,
+  0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45,
+  0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94,
+  0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7,
+  0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a,
+  0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5,
+  0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c,
+  0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1,
+  0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a,
+  0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75,
+  0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051,
+  0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46,
+  0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff,
+  0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77,
+  0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb,
+  0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000,
+  0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e,
+  0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927,
+  0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a,
+  0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e,
+  0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16,
+  0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d,
+  0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8,
+  0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd,
+  0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34,
+  0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163,
+  0xd731dcca, 0x42638510, 0x13972240, 0x84c61120,
+  0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d,
+  0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0,
+  0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422,
+  0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef,
+  0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36,
+  0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4,
+  0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662,
+  0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5,
+  0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3,
+  0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b,
+  0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8,
+  0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6,
+  0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6,
+  0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0,
+  0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815,
+  0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f,
+  0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df,
+  0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f,
+  0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e,
+  0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713,
+  0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89,
+  0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c,
+  0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf,
+  0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86,
+  0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f,
+  0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541,
+  0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190,
+  0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742,
+};
+
+__constant u32 td1[256] =
+{
+  0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e,
+  0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303,
+  0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c,
+  0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3,
+  0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0,
+  0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9,
+  0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259,
+  0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8,
+  0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971,
+  0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a,
+  0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f,
+  0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b,
+  0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8,
+  0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab,
+  0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708,
+  0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682,
+  0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2,
+  0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe,
+  0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb,
+  0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10,
+  0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd,
+  0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015,
+  0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e,
+  0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee,
+  0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000,
+  0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72,
+  0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39,
+  0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e,
+  0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91,
+  0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a,
+  0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17,
+  0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9,
+  0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60,
+  0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e,
+  0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1,
+  0xcad731dc, 0x10426385, 0x40139722, 0x2084c611,
+  0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1,
+  0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3,
+  0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964,
+  0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390,
+  0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b,
+  0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf,
+  0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46,
+  0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af,
+  0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512,
+  0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb,
+  0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a,
+  0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8,
+  0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c,
+  0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266,
+  0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8,
+  0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6,
+  0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604,
+  0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551,
+  0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41,
+  0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647,
+  0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c,
+  0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1,
+  0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737,
+  0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db,
+  0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340,
+  0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95,
+  0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1,
+  0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857,
+};
+
+__constant u32 td2[256] =
+{
+  0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27,
+  0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3,
+  0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502,
+  0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562,
+  0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe,
+  0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3,
+  0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552,
+  0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9,
+  0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9,
+  0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce,
+  0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253,
+  0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908,
+  0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b,
+  0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655,
+  0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337,
+  0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16,
+  0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69,
+  0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6,
+  0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6,
+  0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e,
+  0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6,
+  0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050,
+  0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9,
+  0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8,
+  0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000,
+  0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a,
+  0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d,
+  0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436,
+  0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b,
+  0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12,
+  0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b,
+  0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e,
+  0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f,
+  0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb,
+  0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4,
+  0xdccad731, 0x85104263, 0x22401397, 0x112084c6,
+  0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729,
+  0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1,
+  0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9,
+  0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233,
+  0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4,
+  0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad,
+  0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e,
+  0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3,
+  0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25,
+  0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b,
+  0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f,
+  0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15,
+  0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0,
+  0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2,
+  0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7,
+  0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791,
+  0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496,
+  0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665,
+  0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b,
+  0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6,
+  0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13,
+  0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47,
+  0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7,
+  0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844,
+  0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3,
+  0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d,
+  0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456,
+  0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8,
+};
+
+__constant u32 td3[256] =
+{
+  0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a,
+  0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b,
+  0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5,
+  0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5,
+  0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d,
+  0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b,
+  0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95,
+  0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e,
+  0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27,
+  0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d,
+  0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562,
+  0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9,
+  0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752,
+  0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66,
+  0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3,
+  0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced,
+  0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e,
+  0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4,
+  0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4,
+  0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd,
+  0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d,
+  0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60,
+  0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767,
+  0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79,
+  0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000,
+  0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c,
+  0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736,
+  0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24,
+  0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b,
+  0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c,
+  0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12,
+  0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814,
+  0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3,
+  0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b,
+  0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8,
+  0x31dccad7, 0x63851042, 0x97224013, 0xc6112084,
+  0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7,
+  0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077,
+  0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247,
+  0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22,
+  0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698,
+  0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f,
+  0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254,
+  0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582,
+  0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf,
+  0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb,
+  0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883,
+  0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef,
+  0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629,
+  0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035,
+  0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533,
+  0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17,
+  0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4,
+  0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46,
+  0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb,
+  0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d,
+  0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb,
+  0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a,
+  0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73,
+  0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678,
+  0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2,
+  0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff,
+  0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064,
+  0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0,
+};
+
+__constant u32 td4[256] =
+{
+  0x52525252, 0x09090909, 0x6a6a6a6a, 0xd5d5d5d5,
+  0x30303030, 0x36363636, 0xa5a5a5a5, 0x38383838,
+  0xbfbfbfbf, 0x40404040, 0xa3a3a3a3, 0x9e9e9e9e,
+  0x81818181, 0xf3f3f3f3, 0xd7d7d7d7, 0xfbfbfbfb,
+  0x7c7c7c7c, 0xe3e3e3e3, 0x39393939, 0x82828282,
+  0x9b9b9b9b, 0x2f2f2f2f, 0xffffffff, 0x87878787,
+  0x34343434, 0x8e8e8e8e, 0x43434343, 0x44444444,
+  0xc4c4c4c4, 0xdededede, 0xe9e9e9e9, 0xcbcbcbcb,
+  0x54545454, 0x7b7b7b7b, 0x94949494, 0x32323232,
+  0xa6a6a6a6, 0xc2c2c2c2, 0x23232323, 0x3d3d3d3d,
+  0xeeeeeeee, 0x4c4c4c4c, 0x95959595, 0x0b0b0b0b,
+  0x42424242, 0xfafafafa, 0xc3c3c3c3, 0x4e4e4e4e,
+  0x08080808, 0x2e2e2e2e, 0xa1a1a1a1, 0x66666666,
+  0x28282828, 0xd9d9d9d9, 0x24242424, 0xb2b2b2b2,
+  0x76767676, 0x5b5b5b5b, 0xa2a2a2a2, 0x49494949,
+  0x6d6d6d6d, 0x8b8b8b8b, 0xd1d1d1d1, 0x25252525,
+  0x72727272, 0xf8f8f8f8, 0xf6f6f6f6, 0x64646464,
+  0x86868686, 0x68686868, 0x98989898, 0x16161616,
+  0xd4d4d4d4, 0xa4a4a4a4, 0x5c5c5c5c, 0xcccccccc,
+  0x5d5d5d5d, 0x65656565, 0xb6b6b6b6, 0x92929292,
+  0x6c6c6c6c, 0x70707070, 0x48484848, 0x50505050,
+  0xfdfdfdfd, 0xedededed, 0xb9b9b9b9, 0xdadadada,
+  0x5e5e5e5e, 0x15151515, 0x46464646, 0x57575757,
+  0xa7a7a7a7, 0x8d8d8d8d, 0x9d9d9d9d, 0x84848484,
+  0x90909090, 0xd8d8d8d8, 0xabababab, 0x00000000,
+  0x8c8c8c8c, 0xbcbcbcbc, 0xd3d3d3d3, 0x0a0a0a0a,
+  0xf7f7f7f7, 0xe4e4e4e4, 0x58585858, 0x05050505,
+  0xb8b8b8b8, 0xb3b3b3b3, 0x45454545, 0x06060606,
+  0xd0d0d0d0, 0x2c2c2c2c, 0x1e1e1e1e, 0x8f8f8f8f,
+  0xcacacaca, 0x3f3f3f3f, 0x0f0f0f0f, 0x02020202,
+  0xc1c1c1c1, 0xafafafaf, 0xbdbdbdbd, 0x03030303,
+  0x01010101, 0x13131313, 0x8a8a8a8a, 0x6b6b6b6b,
+  0x3a3a3a3a, 0x91919191, 0x11111111, 0x41414141,
+  0x4f4f4f4f, 0x67676767, 0xdcdcdcdc, 0xeaeaeaea,
+  0x97979797, 0xf2f2f2f2, 0xcfcfcfcf, 0xcececece,
+  0xf0f0f0f0, 0xb4b4b4b4, 0xe6e6e6e6, 0x73737373,
+  0x96969696, 0xacacacac, 0x74747474, 0x22222222,
+  0xe7e7e7e7, 0xadadadad, 0x35353535, 0x85858585,
+  0xe2e2e2e2, 0xf9f9f9f9, 0x37373737, 0xe8e8e8e8,
+  0x1c1c1c1c, 0x75757575, 0xdfdfdfdf, 0x6e6e6e6e,
+  0x47474747, 0xf1f1f1f1, 0x1a1a1a1a, 0x71717171,
+  0x1d1d1d1d, 0x29292929, 0xc5c5c5c5, 0x89898989,
+  0x6f6f6f6f, 0xb7b7b7b7, 0x62626262, 0x0e0e0e0e,
+  0xaaaaaaaa, 0x18181818, 0xbebebebe, 0x1b1b1b1b,
+  0xfcfcfcfc, 0x56565656, 0x3e3e3e3e, 0x4b4b4b4b,
+  0xc6c6c6c6, 0xd2d2d2d2, 0x79797979, 0x20202020,
+  0x9a9a9a9a, 0xdbdbdbdb, 0xc0c0c0c0, 0xfefefefe,
+  0x78787878, 0xcdcdcdcd, 0x5a5a5a5a, 0xf4f4f4f4,
+  0x1f1f1f1f, 0xdddddddd, 0xa8a8a8a8, 0x33333333,
+  0x88888888, 0x07070707, 0xc7c7c7c7, 0x31313131,
+  0xb1b1b1b1, 0x12121212, 0x10101010, 0x59595959,
+  0x27272727, 0x80808080, 0xecececec, 0x5f5f5f5f,
+  0x60606060, 0x51515151, 0x7f7f7f7f, 0xa9a9a9a9,
+  0x19191919, 0xb5b5b5b5, 0x4a4a4a4a, 0x0d0d0d0d,
+  0x2d2d2d2d, 0xe5e5e5e5, 0x7a7a7a7a, 0x9f9f9f9f,
+  0x93939393, 0xc9c9c9c9, 0x9c9c9c9c, 0xefefefef,
+  0xa0a0a0a0, 0xe0e0e0e0, 0x3b3b3b3b, 0x4d4d4d4d,
+  0xaeaeaeae, 0x2a2a2a2a, 0xf5f5f5f5, 0xb0b0b0b0,
+  0xc8c8c8c8, 0xebebebeb, 0xbbbbbbbb, 0x3c3c3c3c,
+  0x83838383, 0x53535353, 0x99999999, 0x61616161,
+  0x17171717, 0x2b2b2b2b, 0x04040404, 0x7e7e7e7e,
+  0xbabababa, 0x77777777, 0xd6d6d6d6, 0x26262626,
+  0xe1e1e1e1, 0x69696969, 0x14141414, 0x63636363,
+  0x55555555, 0x21212121, 0x0c0c0c0c, 0x7d7d7d7d,
+};
+
+__constant u32 rcon[] =
+{
+  0x01000000, 0x02000000, 0x04000000, 0x08000000,
+  0x10000000, 0x20000000, 0x40000000, 0x80000000,
+  0x1b000000, 0x36000000,
+};
+
+void aes256_ExpandKey (u32 *ks, const u32 *ukey)
+{
+  ks[0] = ukey[0];
+  ks[1] = ukey[1];
+  ks[2] = ukey[2];
+  ks[3] = ukey[3];
+  ks[4] = ukey[4];
+  ks[5] = ukey[5];
+  ks[6] = ukey[6];
+  ks[7] = ukey[7];
+
+  int i;
+  int j;
+
+  i = 0;
+  j = 0;
+
+  while (1)
+  {
+    u32 temp = ks[j +  7];
+
+    ks[j +  8] = ks[j +  0]
+           ^ (te2[(temp >> 16) & 0xff] & 0xff000000)
+           ^ (te3[(temp >>  8) & 0xff] & 0x00ff0000)
+           ^ (te0[(temp >>  0) & 0xff] & 0x0000ff00)
+           ^ (te1[(temp >> 24) & 0xff] & 0x000000ff)
+           ^ rcon[i];
+
+    ks[j +  9] = ks[j +  1] ^ ks[j +  8];
+    ks[j + 10] = ks[j +  2] ^ ks[j +  9];
+    ks[j + 11] = ks[j +  3] ^ ks[j + 10];
+
+    if (++i == 7) break;
+
+    temp = ks[j + 11];
+
+    ks[j + 12] = ks[j +  4]
+           ^ (te2[(temp >> 24) & 0xff] & 0xff000000)
+           ^ (te3[(temp >> 16) & 0xff] & 0x00ff0000)
+           ^ (te0[(temp >>  8) & 0xff] & 0x0000ff00)
+           ^ (te1[(temp >>  0) & 0xff] & 0x000000ff);
+
+    ks[j + 13] = ks[j +  5] ^ ks[j + 12];
+    ks[j + 14] = ks[j +  6] ^ ks[j + 13];
+    ks[j + 15] = ks[j +  7] ^ ks[j + 14];
+
+    j += 8;
+  }
+}
+
+void aes256_InvertKey (u32 *ks)
+{
+  for (u32 i = 0, j = 56; i < j; i += 4, j -= 4)
+  {
+    u32 temp;
+
+    temp = ks[i + 0]; ks[i + 0] = ks[j + 0]; ks[j + 0] = temp;
+    temp = ks[i + 1]; ks[i + 1] = ks[j + 1]; ks[j + 1] = temp;
+    temp = ks[i + 2]; ks[i + 2] = ks[j + 2]; ks[j + 2] = temp;
+    temp = ks[i + 3]; ks[i + 3] = ks[j + 3]; ks[j + 3] = temp;
+  }
+
+  for (u32 i = 1, j = 4; i < 14; i += 1, j += 4)
+  {
+    ks[j + 0] =
+      td0[te1[(ks[j + 0] >> 24) & 0xff] & 0xff] ^
+      td1[te1[(ks[j + 0] >> 16) & 0xff] & 0xff] ^
+      td2[te1[(ks[j + 0] >>  8) & 0xff] & 0xff] ^
+      td3[te1[(ks[j + 0] >>  0) & 0xff] & 0xff];
+
+    ks[j + 1] =
+      td0[te1[(ks[j + 1] >> 24) & 0xff] & 0xff] ^
+      td1[te1[(ks[j + 1] >> 16) & 0xff] & 0xff] ^
+      td2[te1[(ks[j + 1] >>  8) & 0xff] & 0xff] ^
+      td3[te1[(ks[j + 1] >>  0) & 0xff] & 0xff];
+
+    ks[j + 2] =
+      td0[te1[(ks[j + 2] >> 24) & 0xff] & 0xff] ^
+      td1[te1[(ks[j + 2] >> 16) & 0xff] & 0xff] ^
+      td2[te1[(ks[j + 2] >>  8) & 0xff] & 0xff] ^
+      td3[te1[(ks[j + 2] >>  0) & 0xff] & 0xff];
+
+    ks[j + 3] =
+      td0[te1[(ks[j + 3] >> 24) & 0xff] & 0xff] ^
+      td1[te1[(ks[j + 3] >> 16) & 0xff] & 0xff] ^
+      td2[te1[(ks[j + 3] >>  8) & 0xff] & 0xff] ^
+      td3[te1[(ks[j + 3] >>  0) & 0xff] & 0xff];
+  }
+}
+
+void aes256_set_encrypt_key (u32 *ks, const u32 *ukey)
+{
+  u32 ukey_s[8];
+
+  ukey_s[0] = swap32 (ukey[0]);
+  ukey_s[1] = swap32 (ukey[1]);
+  ukey_s[2] = swap32 (ukey[2]);
+  ukey_s[3] = swap32 (ukey[3]);
+  ukey_s[4] = swap32 (ukey[4]);
+  ukey_s[5] = swap32 (ukey[5]);
+  ukey_s[6] = swap32 (ukey[6]);
+  ukey_s[7] = swap32 (ukey[7]);
+
+  aes256_ExpandKey (ks, ukey_s);
+}
+
+void aes256_set_decrypt_key (u32 *ks, const u32 *ukey)
+{
+  u32 ukey_s[8];
+
+  ukey_s[0] = swap32 (ukey[0]);
+  ukey_s[1] = swap32 (ukey[1]);
+  ukey_s[2] = swap32 (ukey[2]);
+  ukey_s[3] = swap32 (ukey[3]);
+  ukey_s[4] = swap32 (ukey[4]);
+  ukey_s[5] = swap32 (ukey[5]);
+  ukey_s[6] = swap32 (ukey[6]);
+  ukey_s[7] = swap32 (ukey[7]);
+
+  aes256_ExpandKey (ks, ukey_s);
+
+  aes256_InvertKey (ks);
+}
+
+void aes256_decrypt (const u32 *ks, const u32 *in, u32 *out)
+{
+  u32 in_s[4];
+
+  in_s[0] = swap32 (in[0]);
+  in_s[1] = swap32 (in[1]);
+  in_s[2] = swap32 (in[2]);
+  in_s[3] = swap32 (in[3]);
+
+  u32 s0 = in_s[0] ^ ks[0];
+  u32 s1 = in_s[1] ^ ks[1];
+  u32 s2 = in_s[2] ^ ks[2];
+  u32 s3 = in_s[3] ^ ks[3];
+
+  u32 t0;
+  u32 t1;
+  u32 t2;
+  u32 t3;
+
+  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[ 4];
+  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[ 5];
+  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[ 6];
+  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[ 7];
+  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[ 8];
+  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[ 9];
+  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[10];
+  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[11];
+  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[12];
+  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[13];
+  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[14];
+  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[15];
+  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[16];
+  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[17];
+  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[18];
+  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[19];
+  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[20];
+  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[21];
+  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[22];
+  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[23];
+  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[24];
+  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[25];
+  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[26];
+  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[27];
+  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[28];
+  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[29];
+  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[30];
+  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[31];
+  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[32];
+  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[33];
+  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[34];
+  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[35];
+  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[36];
+  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[37];
+  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[38];
+  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[39];
+  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[40];
+  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[41];
+  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[42];
+  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[43];
+  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[44];
+  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[45];
+  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[46];
+  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[47];
+  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[48];
+  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[49];
+  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[50];
+  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[51];
+  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[52];
+  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[53];
+  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[54];
+  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[55];
+
+  out[0] = (td4[(t0 >> 24) & 0xff] & 0xff000000)
+         ^ (td4[(t3 >> 16) & 0xff] & 0x00ff0000)
+         ^ (td4[(t2 >>  8) & 0xff] & 0x0000ff00)
+         ^ (td4[(t1 >>  0) & 0xff] & 0x000000ff)
+         ^ ks[56];
+
+  out[1] = (td4[(t1 >> 24) & 0xff] & 0xff000000)
+         ^ (td4[(t0 >> 16) & 0xff] & 0x00ff0000)
+         ^ (td4[(t3 >>  8) & 0xff] & 0x0000ff00)
+         ^ (td4[(t2 >>  0) & 0xff] & 0x000000ff)
+         ^ ks[57];
+
+  out[2] = (td4[(t2 >> 24) & 0xff] & 0xff000000)
+         ^ (td4[(t1 >> 16) & 0xff] & 0x00ff0000)
+         ^ (td4[(t0 >>  8) & 0xff] & 0x0000ff00)
+         ^ (td4[(t3 >>  0) & 0xff] & 0x000000ff)
+         ^ ks[58];
+
+  out[3] = (td4[(t3 >> 24) & 0xff] & 0xff000000)
+         ^ (td4[(t2 >> 16) & 0xff] & 0x00ff0000)
+         ^ (td4[(t1 >>  8) & 0xff] & 0x0000ff00)
+         ^ (td4[(t0 >>  0) & 0xff] & 0x000000ff)
+         ^ ks[59];
+
+  out[0] = swap32 (out[0]);
+  out[1] = swap32 (out[1]);
+  out[2] = swap32 (out[2]);
+  out[3] = swap32 (out[3]);
+}
+
+void aes256_encrypt (const u32 *ks, const u32 *in, u32 *out)
+{
+  u32 in_s[4];
+
+  in_s[0] = swap32 (in[0]);
+  in_s[1] = swap32 (in[1]);
+  in_s[2] = swap32 (in[2]);
+  in_s[3] = swap32 (in[3]);
+
+  u32 s0 = in_s[0] ^ ks[0];
+  u32 s1 = in_s[1] ^ ks[1];
+  u32 s2 = in_s[2] ^ ks[2];
+  u32 s3 = in_s[3] ^ ks[3];
+
+  u32 t0;
+  u32 t1;
+  u32 t2;
+  u32 t3;
+
+  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[ 4];
+  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[ 5];
+  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[ 6];
+  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[ 7];
+  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[ 8];
+  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[ 9];
+  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[10];
+  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[11];
+  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[12];
+  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[13];
+  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[14];
+  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[15];
+  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[16];
+  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[17];
+  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[18];
+  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[19];
+  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[20];
+  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[21];
+  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[22];
+  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[23];
+  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[24];
+  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[25];
+  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[26];
+  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[27];
+  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[28];
+  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[29];
+  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[30];
+  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[31];
+  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[32];
+  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[33];
+  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[34];
+  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[35];
+  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[36];
+  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[37];
+  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[38];
+  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[39];
+  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[40];
+  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[41];
+  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[42];
+  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[43];
+  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[44];
+  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[45];
+  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[46];
+  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[47];
+  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[48];
+  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[49];
+  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[50];
+  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[51];
+  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[52];
+  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[53];
+  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[54];
+  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[55];
+
+  out[0] = (te4[(t0 >> 24) & 0xff] & 0xff000000)
+         ^ (te4[(t1 >> 16) & 0xff] & 0x00ff0000)
+         ^ (te4[(t2 >>  8) & 0xff] & 0x0000ff00)
+         ^ (te4[(t3 >>  0) & 0xff] & 0x000000ff)
+         ^ ks[56];
+
+  out[1] = (te4[(t1 >> 24) & 0xff] & 0xff000000)
+         ^ (te4[(t2 >> 16) & 0xff] & 0x00ff0000)
+         ^ (te4[(t3 >>  8) & 0xff] & 0x0000ff00)
+         ^ (te4[(t0 >>  0) & 0xff] & 0x000000ff)
+         ^ ks[57];
+
+  out[2] = (te4[(t2 >> 24) & 0xff] & 0xff000000)
+         ^ (te4[(t3 >> 16) & 0xff] & 0x00ff0000)
+         ^ (te4[(t0 >>  8) & 0xff] & 0x0000ff00)
+         ^ (te4[(t1 >>  0) & 0xff] & 0x000000ff)
+         ^ ks[58];
+
+  out[3] = (te4[(t3 >> 24) & 0xff] & 0xff000000)
+         ^ (te4[(t0 >> 16) & 0xff] & 0x00ff0000)
+         ^ (te4[(t1 >>  8) & 0xff] & 0x0000ff00)
+         ^ (te4[(t2 >>  0) & 0xff] & 0x000000ff)
+         ^ ks[59];
+
+  out[0] = swap32 (out[0]);
+  out[1] = swap32 (out[1]);
+  out[2] = swap32 (out[2]);
+  out[3] = swap32 (out[3]);
+}
+
+void aes256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out)
+{
+  u32 T[4] = { 0 };
+  u32 Z[4] = { 0 };
+
+  out[0] = in[0];
+  out[1] = in[1];
+  out[2] = in[2];
+  out[3] = in[3];
+
+  u32 ks[60];
+
+  aes256_set_encrypt_key (ks, ukey2);
+  aes256_encrypt (ks, Z, T);
+
+  out[0] ^= T[0];
+  out[1] ^= T[1];
+  out[2] ^= T[2];
+  out[3] ^= T[3];
+
+  aes256_set_decrypt_key (ks, ukey1);
+  aes256_decrypt (ks, out, out);
+
+  out[0] ^= T[0];
+  out[1] ^= T[1];
+  out[2] ^= T[2];
+  out[3] ^= T[3];
+}

diff --git a/OpenCL/inc_cipher_serpent256.cl b/OpenCL/inc_cipher_serpent256.cl
new file mode 100644 (file)
index 0000000..462c0b9
--- /dev/null
+++ b/OpenCL/inc_cipher_serpent256.cl
@@ -0,0 +1,587 @@
+/* This is an independent implementation of the encryption algorithm:   */
+/*                                                                      */
+/*         Serpent by Ross Anderson, Eli Biham and Lars Knudsen         */
+/*                                                                      */
+/* which is a candidate algorithm in the Advanced Encryption Standard   */
+/* programme of the US National Institute of Standards and Technology.  */
+/*                                                                      */
+/* Copyright in this implementation is held by Dr B R Gladman but I     */
+/* hereby give permission for its free direct or derivative use subject */
+/* to acknowledgment of its origin and compliance with any conditions   */
+/* that the originators of the algorithm place on its exploitation.     */
+/*                                                                      */
+/* Dr Brian Gladman (gladman@seven77.demon.co.uk) 14th January 1999     */
+/*                                                                      */
+/* -------------------------------------------------------------------- */
+/*                                                                      */
+/* Cleaned and optimized for GPU use with hashcat by Jens Steube        */
+
+/* 15 terms */
+
+#define sb0(a,b,c,d,e,f,g,h)    \
+    t1 = a ^ d;     \
+    t2 = a & d;     \
+    t3 = c ^ t1;    \
+    t6 = b & t1;    \
+    t4 = b ^ t3;    \
+    t10 = ~t3;      \
+    h = t2 ^ t4;    \
+    t7 = a ^ t6;    \
+    t14 = ~t7;      \
+    t8 = c | t7;    \
+    t11 = t3 ^ t7;  \
+    g = t4 ^ t8;    \
+    t12 = h & t11;  \
+    f = t10 ^ t12;  \
+    e = t12 ^ t14
+
+/* 15 terms */
+
+#define ib0(a,b,c,d,e,f,g,h)    \
+    t1 = ~a;        \
+    t2 = a ^ b;     \
+    t3 = t1 | t2;   \
+    t4 = d ^ t3;    \
+    t7 = d & t2;    \
+    t5 = c ^ t4;    \
+    t8 = t1 ^ t7;   \
+    g = t2 ^ t5;    \
+    t11 = a & t4;   \
+    t9 = g & t8;    \
+    t14 = t5 ^ t8;  \
+    f = t4 ^ t9;    \
+    t12 = t5 | f;   \
+    h = t11 ^ t12;  \
+    e = h ^ t14
+
+/* 14 terms!  */
+
+#define sb1(a,b,c,d,e,f,g,h)    \
+    t1 = ~a;        \
+    t2 = b ^ t1;    \
+    t3 = a | t2;    \
+    t4 = d | t2;    \
+    t5 = c ^ t3;    \
+    g = d ^ t5;     \
+    t7 = b ^ t4;    \
+    t8 = t2 ^ g;    \
+    t9 = t5 & t7;   \
+    h = t8 ^ t9;    \
+    t11 = t5 ^ t7;  \
+    f = h ^ t11;    \
+    t13 = t8 & t11; \
+    e = t5 ^ t13
+
+/* 17 terms */
+
+#define ib1(a,b,c,d,e,f,g,h)    \
+    t1 = a ^ d;     \
+    t2 = a & b;     \
+    t3 = b ^ c;     \
+    t4 = a ^ t3;    \
+    t5 = b | d;     \
+    t7 = c | t1;    \
+    h = t4 ^ t5;    \
+    t8 = b ^ t7;    \
+    t11 = ~t2;      \
+    t9 = t4 & t8;   \
+    f = t1 ^ t9;    \
+    t13 = t9 ^ t11; \
+    t12 = h & f;    \
+    g = t12 ^ t13;  \
+    t15 = a & d;    \
+    t16 = c ^ t13;  \
+    e = t15 ^ t16
+
+/* 16 terms */
+
+#define sb2(a,b,c,d,e,f,g,h)    \
+    t1 = ~a;        \
+    t2 = b ^ d;     \
+    t3 = c & t1;    \
+    t13 = d | t1;   \
+    e = t2 ^ t3;    \
+    t5 = c ^ t1;    \
+    t6 = c ^ e;     \
+    t7 = b & t6;    \
+    t10 = e | t5;   \
+    h = t5 ^ t7;    \
+    t9 = d | t7;    \
+    t11 = t9 & t10; \
+    t14 = t2 ^ h;   \
+    g = a ^ t11;    \
+    t15 = g ^ t13;  \
+    f = t14 ^ t15
+
+/* 16 terms */
+
+#define ib2(a,b,c,d,e,f,g,h)    \
+    t1 = b ^ d;     \
+    t2 = ~t1;       \
+    t3 = a ^ c;     \
+    t4 = c ^ t1;    \
+    t7 = a | t2;    \
+    t5 = b & t4;    \
+    t8 = d ^ t7;    \
+    t11 = ~t4;      \
+    e = t3 ^ t5;    \
+    t9 = t3 | t8;   \
+    t14 = d & t11;  \
+    h = t1 ^ t9;    \
+    t12 = e | h;    \
+    f = t11 ^ t12;  \
+    t15 = t3 ^ t12; \
+    g = t14 ^ t15
+
+/* 17 terms */
+
+#define sb3(a,b,c,d,e,f,g,h)    \
+    t1 = a ^ c;     \
+    t2 = d ^ t1;    \
+    t3 = a & t2;    \
+    t4 = d ^ t3;    \
+    t5 = b & t4;    \
+    g = t2 ^ t5;    \
+    t7 = a | g;     \
+    t8 = b | d;     \
+    t11 = a | d;    \
+    t9 = t4 & t7;   \
+    f = t8 ^ t9;    \
+    t12 = b ^ t11;  \
+    t13 = g ^ t9;   \
+    t15 = t3 ^ t8;  \
+    h = t12 ^ t13;  \
+    t16 = c & t15;  \
+    e = t12 ^ t16
+
+/* 16 term solution that performs less well than 17 term one
+   in my environment (PPro/PII)
+
+#define sb3(a,b,c,d,e,f,g,h)    \
+    t1 = a ^ b;     \
+    t2 = a & c;     \
+    t3 = a | d;     \
+    t4 = c ^ d;     \
+    t5 = t1 & t3;   \
+    t6 = t2 | t5;   \
+    g = t4 ^ t6;    \
+    t8 = b ^ t3;    \
+    t9 = t6 ^ t8;   \
+    t10 = t4 & t9;  \
+    e = t1 ^ t10;   \
+    t12 = g & e;    \
+    f = t9 ^ t12;   \
+    t14 = b | d;    \
+    t15 = t4 ^ t12; \
+    h = t14 ^ t15
+*/
+
+/* 17 terms */
+
+#define ib3(a,b,c,d,e,f,g,h)    \
+    t1 = b ^ c;     \
+    t2 = b | c;     \
+    t3 = a ^ c;     \
+    t7 = a ^ d;     \
+    t4 = t2 ^ t3;   \
+    t5 = d | t4;    \
+    t9 = t2 ^ t7;   \
+    e = t1 ^ t5;    \
+    t8 = t1 | t5;   \
+    t11 = a & t4;   \
+    g = t8 ^ t9;    \
+    t12 = e | t9;   \
+    f = t11 ^ t12;  \
+    t14 = a & g;    \
+    t15 = t2 ^ t14; \
+    t16 = e & t15;  \
+    h = t4 ^ t16
+
+/* 15 terms */
+
+#define sb4(a,b,c,d,e,f,g,h)    \
+    t1 = a ^ d;     \
+    t2 = d & t1;    \
+    t3 = c ^ t2;    \
+    t4 = b | t3;    \
+    h = t1 ^ t4;    \
+    t6 = ~b;        \
+    t7 = t1 | t6;   \
+    e = t3 ^ t7;    \
+    t9 = a & e;     \
+    t10 = t1 ^ t6;  \
+    t11 = t4 & t10; \
+    g = t9 ^ t11;   \
+    t13 = a ^ t3;   \
+    t14 = t10 & g;  \
+    f = t13 ^ t14
+
+/* 17 terms */
+
+#define ib4(a,b,c,d,e,f,g,h)    \
+    t1 = c ^ d;     \
+    t2 = c | d;     \
+    t3 = b ^ t2;    \
+    t4 = a & t3;    \
+    f = t1 ^ t4;    \
+    t6 = a ^ d;     \
+    t7 = b | d;     \
+    t8 = t6 & t7;   \
+    h = t3 ^ t8;    \
+    t10 = ~a;       \
+    t11 = c ^ h;    \
+    t12 = t10 | t11;\
+    e = t3 ^ t12;   \
+    t14 = c | t4;   \
+    t15 = t7 ^ t14; \
+    t16 = h | t10;  \
+    g = t15 ^ t16
+
+/* 16 terms */
+
+#define sb5(a,b,c,d,e,f,g,h)    \
+    t1 = ~a;        \
+    t2 = a ^ b;     \
+    t3 = a ^ d;     \
+    t4 = c ^ t1;    \
+    t5 = t2 | t3;   \
+    e = t4 ^ t5;    \
+    t7 = d & e;     \
+    t8 = t2 ^ e;    \
+    t10 = t1 | e;   \
+    f = t7 ^ t8;    \
+    t11 = t2 | t7;  \
+    t12 = t3 ^ t10; \
+    t14 = b ^ t7;   \
+    g = t11 ^ t12;  \
+    t15 = f & t12;  \
+    h = t14 ^ t15
+
+/* 16 terms */
+
+#define ib5(a,b,c,d,e,f,g,h)    \
+    t1 = ~c;        \
+    t2 = b & t1;    \
+    t3 = d ^ t2;    \
+    t4 = a & t3;    \
+    t5 = b ^ t1;    \
+    h = t4 ^ t5;    \
+    t7 = b | h;     \
+    t8 = a & t7;    \
+    f = t3 ^ t8;    \
+    t10 = a | d;    \
+    t11 = t1 ^ t7;  \
+    e = t10 ^ t11;  \
+    t13 = a ^ c;    \
+    t14 = b & t10;  \
+    t15 = t4 | t13; \
+    g = t14 ^ t15
+
+/* 15 terms */
+
+#define sb6(a,b,c,d,e,f,g,h)    \
+    t1 = ~a;        \
+    t2 = a ^ d;     \
+    t3 = b ^ t2;    \
+    t4 = t1 | t2;   \
+    t5 = c ^ t4;    \
+    f = b ^ t5;     \
+    t13 = ~t5;      \
+    t7 = t2 | f;    \
+    t8 = d ^ t7;    \
+    t9 = t5 & t8;   \
+    g = t3 ^ t9;    \
+    t11 = t5 ^ t8;  \
+    e = g ^ t11;    \
+    t14 = t3 & t11; \
+    h = t13 ^ t14
+
+/* 15 terms */
+
+#define ib6(a,b,c,d,e,f,g,h)    \
+    t1 = ~a;        \
+    t2 = a ^ b;     \
+    t3 = c ^ t2;    \
+    t4 = c | t1;    \
+    t5 = d ^ t4;    \
+    t13 = d & t1;   \
+    f = t3 ^ t5;    \
+    t7 = t3 & t5;   \
+    t8 = t2 ^ t7;   \
+    t9 = b | t8;    \
+    h = t5 ^ t9;    \
+    t11 = b | h;    \
+    e = t8 ^ t11;   \
+    t14 = t3 ^ t11; \
+    g = t13 ^ t14
+
+/* 17 terms */
+
+#define sb7(a,b,c,d,e,f,g,h)    \
+    t1 = ~c;        \
+    t2 = b ^ c;     \
+    t3 = b | t1;    \
+    t4 = d ^ t3;    \
+    t5 = a & t4;    \
+    t7 = a ^ d;     \
+    h = t2 ^ t5;    \
+    t8 = b ^ t5;    \
+    t9 = t2 | t8;   \
+    t11 = d & t3;   \
+    f = t7 ^ t9;    \
+    t12 = t5 ^ f;   \
+    t15 = t1 | t4;  \
+    t13 = h & t12;  \
+    g = t11 ^ t13;  \
+    t16 = t12 ^ g;  \
+    e = t15 ^ t16
+
+/* 17 terms */
+
+#define ib7(a,b,c,d,e,f,g,h)    \
+    t1 = a & b;     \
+    t2 = a | b;     \
+    t3 = c | t1;    \
+    t4 = d & t2;    \
+    h = t3 ^ t4;    \
+    t6 = ~d;        \
+    t7 = b ^ t4;    \
+    t8 = h ^ t6;    \
+    t11 = c ^ t7;   \
+    t9 = t7 | t8;   \
+    f = a ^ t9;     \
+    t12 = d | f;    \
+    e = t11 ^ t12;  \
+    t14 = a & h;    \
+    t15 = t3 ^ f;   \
+    t16 = e ^ t14;  \
+    g = t15 ^ t16
+
+#define k_xor(r,a,b,c,d)    \
+    a ^= ks[4 * r +  8]; \
+    b ^= ks[4 * r +  9]; \
+    c ^= ks[4 * r + 10]; \
+    d ^= ks[4 * r + 11]
+
+#define k_set(r,a,b,c,d)    \
+    a = ks[4 * r +  8];  \
+    b = ks[4 * r +  9];  \
+    c = ks[4 * r + 10];  \
+    d = ks[4 * r + 11]
+
+#define k_get(r,a,b,c,d)    \
+    ks[4 * r +  8] = a;  \
+    ks[4 * r +  9] = b;  \
+    ks[4 * r + 10] = c;  \
+    ks[4 * r + 11] = d
+
+/* the linear transformation and its inverse    */
+
+#define rot(a,b,c,d)    \
+    a = rotl32(a, 13);    \
+    c = rotl32(c, 3);     \
+    d ^= c ^ (a << 3);  \
+    b ^= a ^ c;         \
+    d = rotl32(d, 7);     \
+    b = rotl32(b, 1);     \
+    a ^= b ^ d;         \
+    c ^= d ^ (b << 7);  \
+    a = rotl32(a, 5);     \
+    c = rotl32(c, 22)
+
+#define irot(a,b,c,d)   \
+    c = rotr32(c, 22);    \
+    a = rotr32(a, 5);     \
+    c ^= d ^ (b << 7);  \
+    a ^= b ^ d;         \
+    d = rotr32(d, 7);     \
+    b = rotr32(b, 1);     \
+    d ^= c ^ (a << 3);  \
+    b ^= a ^ c;         \
+    c = rotr32(c, 3);     \
+    a = rotr32(a, 13)
+
+void serpent256_set_key (u32 *ks, const u32 *ukey)
+{
+  #ifdef _unroll
+  #pragma unroll
+  #endif
+  for (int i = 0; i < 8; i++)
+  {
+    ks[i] = ukey[i];
+  }
+
+  #ifdef _unroll
+  #pragma unroll
+  #endif
+  for (int i = 0; i < 132; i++)
+  {
+    ks[i + 8] = rotl32 (ks[i + 7] ^ ks[i + 5] ^ ks[i + 3] ^ ks[i + 0] ^ 0x9e3779b9 ^ i, 11);
+  }
+
+  u32  a,b,c,d,e,f,g,h;
+  u32  t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16;
+
+  k_set( 0,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get( 0,e,f,g,h);
+  k_set( 1,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get( 1,e,f,g,h);
+  k_set( 2,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get( 2,e,f,g,h);
+  k_set( 3,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get( 3,e,f,g,h);
+  k_set( 4,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get( 4,e,f,g,h);
+  k_set( 5,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get( 5,e,f,g,h);
+  k_set( 6,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get( 6,e,f,g,h);
+  k_set( 7,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get( 7,e,f,g,h);
+  k_set( 8,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get( 8,e,f,g,h);
+  k_set( 9,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get( 9,e,f,g,h);
+  k_set(10,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(10,e,f,g,h);
+  k_set(11,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(11,e,f,g,h);
+  k_set(12,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(12,e,f,g,h);
+  k_set(13,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(13,e,f,g,h);
+  k_set(14,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(14,e,f,g,h);
+  k_set(15,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(15,e,f,g,h);
+  k_set(16,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(16,e,f,g,h);
+  k_set(17,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get(17,e,f,g,h);
+  k_set(18,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(18,e,f,g,h);
+  k_set(19,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(19,e,f,g,h);
+  k_set(20,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(20,e,f,g,h);
+  k_set(21,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(21,e,f,g,h);
+  k_set(22,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(22,e,f,g,h);
+  k_set(23,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(23,e,f,g,h);
+  k_set(24,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(24,e,f,g,h);
+  k_set(25,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get(25,e,f,g,h);
+  k_set(26,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(26,e,f,g,h);
+  k_set(27,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(27,e,f,g,h);
+  k_set(28,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(28,e,f,g,h);
+  k_set(29,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(29,e,f,g,h);
+  k_set(30,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(30,e,f,g,h);
+  k_set(31,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(31,e,f,g,h);
+  k_set(32,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(32,e,f,g,h);
+}
+
+void serpent256_encrypt (const u32 *ks, const u32 *in, u32 *out)
+{
+  u32  a,b,c,d,e,f,g,h;
+  u32  t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16;
+
+  a = in[0];
+  b = in[1];
+  c = in[2];
+  d = in[3];
+
+  k_xor( 0,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor( 1,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor( 2,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor( 3,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor( 4,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor( 5,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor( 6,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor( 7,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor( 8,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor( 9,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(10,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(11,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(12,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(13,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(14,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(15,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(16,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(17,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(18,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(19,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(20,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(21,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(22,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(23,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(24,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(25,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(26,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(27,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(28,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(29,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d);
+  k_xor(30,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h);
+  k_xor(31,e,f,g,h); sb7(e,f,g,h,a,b,c,d);
+  k_xor(32,a,b,c,d);
+
+  out[0] = a;
+  out[1] = b;
+  out[2] = c;
+  out[3] = d;
+}
+
+void serpent256_decrypt (const u32 *ks, const u32 *in, u32 *out)
+{
+  u32  a,b,c,d,e,f,g,h;
+  u32  t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16;
+
+  a = in[0];
+  b = in[1];
+  c = in[2];
+  d = in[3];
+
+                                       k_xor(32,a,b,c,d);
+                 ib7(a,b,c,d,e,f,g,h); k_xor(31,e,f,g,h);
+  irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(30,a,b,c,d);
+  irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(29,e,f,g,h);
+  irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(28,a,b,c,d);
+  irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(27,e,f,g,h);
+  irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(26,a,b,c,d);
+  irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor(25,e,f,g,h);
+  irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor(24,a,b,c,d);
+  irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor(23,e,f,g,h);
+  irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(22,a,b,c,d);
+  irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(21,e,f,g,h);
+  irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(20,a,b,c,d);
+  irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(19,e,f,g,h);
+  irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(18,a,b,c,d);
+  irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor(17,e,f,g,h);
+  irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor(16,a,b,c,d);
+  irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor(15,e,f,g,h);
+  irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(14,a,b,c,d);
+  irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(13,e,f,g,h);
+  irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(12,a,b,c,d);
+  irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(11,e,f,g,h);
+  irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(10,a,b,c,d);
+  irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor( 9,e,f,g,h);
+  irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor( 8,a,b,c,d);
+  irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor( 7,e,f,g,h);
+  irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor( 6,a,b,c,d);
+  irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor( 5,e,f,g,h);
+  irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor( 4,a,b,c,d);
+  irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor( 3,e,f,g,h);
+  irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor( 2,a,b,c,d);
+  irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor( 1,e,f,g,h);
+  irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor( 0,a,b,c,d);
+
+  out[0] = a;
+  out[1] = b;
+  out[2] = c;
+  out[3] = d;
+}
+
+void serpent256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out)
+{
+  u32 T[4] = { 0 };
+  u32 Z[4] = { 0 };
+
+  out[0] = in[0];
+  out[1] = in[1];
+  out[2] = in[2];
+  out[3] = in[3];
+
+  u32 ks[140];
+
+  serpent256_set_key (ks, ukey2);
+  serpent256_encrypt (ks, Z, T);
+
+  out[0] ^= T[0];
+  out[1] ^= T[1];
+  out[2] ^= T[2];
+  out[3] ^= T[3];
+
+  serpent256_set_key (ks, ukey1);
+  serpent256_decrypt (ks, out, out);
+
+  out[0] ^= T[0];
+  out[1] ^= T[1];
+  out[2] ^= T[2];
+  out[3] ^= T[3];
+}

diff --git a/OpenCL/inc_cipher_twofish256.cl b/OpenCL/inc_cipher_twofish256.cl
new file mode 100644 (file)
index 0000000..6482a63
--- /dev/null
+++ b/OpenCL/inc_cipher_twofish256.cl
@@ -0,0 +1,466 @@
+/* This is an independent implementation of the encryption algorithm:   */
+/*                                                                      */
+/*         Twofish by Bruce Schneier and colleagues                     */
+/*                                                                      */
+/* which is a candidate algorithm in the Advanced Encryption Standard   */
+/* programme of the US National Institute of Standards and Technology.  */
+/*                                                                      */
+/* Copyright in this implementation is held by Dr B R Gladman but I     */
+/* hereby give permission for its free direct or derivative use subject */
+/* to acknowledgment of its origin and compliance with any conditions   */
+/* that the originators of t he algorithm place on its exploitation.    */
+/*                                                                      */
+/* My thanks to Doug Whiting and Niels Ferguson for comments that led   */
+/* to improvements in this implementation.                              */
+/*                                                                      */
+/* Dr Brian Gladman (gladman@seven77.demon.co.uk) 14th January 1999     */
+/*                                                                      */
+/* -------------------------------------------------------------------- */
+/*                                                                      */
+/* Cleaned and optimized for GPU use with hashcat by Jens Steube        */
+
+#define extract_byte(x,n) (((x) >> (8 * (n))) & 0xff)
+
+__constant u32 q_tab[2][256] =
+{
+  {
+    0xA9, 0x67, 0xB3, 0xE8, 0x04, 0xFD, 0xA3, 0x76, 0x9A, 0x92, 0x80, 0x78,
+    0xE4, 0xDD, 0xD1, 0x38, 0x0D, 0xC6, 0x35, 0x98, 0x18, 0xF7, 0xEC, 0x6C,
+    0x43, 0x75, 0x37, 0x26, 0xFA, 0x13, 0x94, 0x48, 0xF2, 0xD0, 0x8B, 0x30,
+    0x84, 0x54, 0xDF, 0x23, 0x19, 0x5B, 0x3D, 0x59, 0xF3, 0xAE, 0xA2, 0x82,
+    0x63, 0x01, 0x83, 0x2E, 0xD9, 0x51, 0x9B, 0x7C, 0xA6, 0xEB, 0xA5, 0xBE,
+    0x16, 0x0C, 0xE3, 0x61, 0xC0, 0x8C, 0x3A, 0xF5, 0x73, 0x2C, 0x25, 0x0B,
+    0xBB, 0x4E, 0x89, 0x6B, 0x53, 0x6A, 0xB4, 0xF1, 0xE1, 0xE6, 0xBD, 0x45,
+    0xE2, 0xF4, 0xB6, 0x66, 0xCC, 0x95, 0x03, 0x56, 0xD4, 0x1C, 0x1E, 0xD7,
+    0xFB, 0xC3, 0x8E, 0xB5, 0xE9, 0xCF, 0xBF, 0xBA, 0xEA, 0x77, 0x39, 0xAF,
+    0x33, 0xC9, 0x62, 0x71, 0x81, 0x79, 0x09, 0xAD, 0x24, 0xCD, 0xF9, 0xD8,
+    0xE5, 0xC5, 0xB9, 0x4D, 0x44, 0x08, 0x86, 0xE7, 0xA1, 0x1D, 0xAA, 0xED,
+    0x06, 0x70, 0xB2, 0xD2, 0x41, 0x7B, 0xA0, 0x11, 0x31, 0xC2, 0x27, 0x90,
+    0x20, 0xF6, 0x60, 0xFF, 0x96, 0x5C, 0xB1, 0xAB, 0x9E, 0x9C, 0x52, 0x1B,
+    0x5F, 0x93, 0x0A, 0xEF, 0x91, 0x85, 0x49, 0xEE, 0x2D, 0x4F, 0x8F, 0x3B,
+    0x47, 0x87, 0x6D, 0x46, 0xD6, 0x3E, 0x69, 0x64, 0x2A, 0xCE, 0xCB, 0x2F,
+    0xFC, 0x97, 0x05, 0x7A, 0xAC, 0x7F, 0xD5, 0x1A, 0x4B, 0x0E, 0xA7, 0x5A,
+    0x28, 0x14, 0x3F, 0x29, 0x88, 0x3C, 0x4C, 0x02, 0xB8, 0xDA, 0xB0, 0x17,
+    0x55, 0x1F, 0x8A, 0x7D, 0x57, 0xC7, 0x8D, 0x74, 0xB7, 0xC4, 0x9F, 0x72,
+    0x7E, 0x15, 0x22, 0x12, 0x58, 0x07, 0x99, 0x34, 0x6E, 0x50, 0xDE, 0x68,
+    0x65, 0xBC, 0xDB, 0xF8, 0xC8, 0xA8, 0x2B, 0x40, 0xDC, 0xFE, 0x32, 0xA4,
+    0xCA, 0x10, 0x21, 0xF0, 0xD3, 0x5D, 0x0F, 0x00, 0x6F, 0x9D, 0x36, 0x42,
+    0x4A, 0x5E, 0xC1, 0xE0
+  },
+  {
+    0x75, 0xF3, 0xC6, 0xF4, 0xDB, 0x7B, 0xFB, 0xC8, 0x4A, 0xD3, 0xE6, 0x6B,
+    0x45, 0x7D, 0xE8, 0x4B, 0xD6, 0x32, 0xD8, 0xFD, 0x37, 0x71, 0xF1, 0xE1,
+    0x30, 0x0F, 0xF8, 0x1B, 0x87, 0xFA, 0x06, 0x3F, 0x5E, 0xBA, 0xAE, 0x5B,
+    0x8A, 0x00, 0xBC, 0x9D, 0x6D, 0xC1, 0xB1, 0x0E, 0x80, 0x5D, 0xD2, 0xD5,
+    0xA0, 0x84, 0x07, 0x14, 0xB5, 0x90, 0x2C, 0xA3, 0xB2, 0x73, 0x4C, 0x54,
+    0x92, 0x74, 0x36, 0x51, 0x38, 0xB0, 0xBD, 0x5A, 0xFC, 0x60, 0x62, 0x96,
+    0x6C, 0x42, 0xF7, 0x10, 0x7C, 0x28, 0x27, 0x8C, 0x13, 0x95, 0x9C, 0xC7,
+    0x24, 0x46, 0x3B, 0x70, 0xCA, 0xE3, 0x85, 0xCB, 0x11, 0xD0, 0x93, 0xB8,
+    0xA6, 0x83, 0x20, 0xFF, 0x9F, 0x77, 0xC3, 0xCC, 0x03, 0x6F, 0x08, 0xBF,
+    0x40, 0xE7, 0x2B, 0xE2, 0x79, 0x0C, 0xAA, 0x82, 0x41, 0x3A, 0xEA, 0xB9,
+    0xE4, 0x9A, 0xA4, 0x97, 0x7E, 0xDA, 0x7A, 0x17, 0x66, 0x94, 0xA1, 0x1D,
+    0x3D, 0xF0, 0xDE, 0xB3, 0x0B, 0x72, 0xA7, 0x1C, 0xEF, 0xD1, 0x53, 0x3E,
+    0x8F, 0x33, 0x26, 0x5F, 0xEC, 0x76, 0x2A, 0x49, 0x81, 0x88, 0xEE, 0x21,
+    0xC4, 0x1A, 0xEB, 0xD9, 0xC5, 0x39, 0x99, 0xCD, 0xAD, 0x31, 0x8B, 0x01,
+    0x18, 0x23, 0xDD, 0x1F, 0x4E, 0x2D, 0xF9, 0x48, 0x4F, 0xF2, 0x65, 0x8E,
+    0x78, 0x5C, 0x58, 0x19, 0x8D, 0xE5, 0x98, 0x57, 0x67, 0x7F, 0x05, 0x64,
+    0xAF, 0x63, 0xB6, 0xFE, 0xF5, 0xB7, 0x3C, 0xA5, 0xCE, 0xE9, 0x68, 0x44,
+    0xE0, 0x4D, 0x43, 0x69, 0x29, 0x2E, 0xAC, 0x15, 0x59, 0xA8, 0x0A, 0x9E,
+    0x6E, 0x47, 0xDF, 0x34, 0x35, 0x6A, 0xCF, 0xDC, 0x22, 0xC9, 0xC0, 0x9B,
+    0x89, 0xD4, 0xED, 0xAB, 0x12, 0xA2, 0x0D, 0x52, 0xBB, 0x02, 0x2F, 0xA9,
+    0xD7, 0x61, 0x1E, 0xB4, 0x50, 0x04, 0xF6, 0xC2, 0x16, 0x25, 0x86, 0x56,
+    0x55, 0x09, 0xBE, 0x91
+  }
+};
+
+#define q(n,x) q_tab[n][x]
+
+__constant u32  m_tab[4][256] =
+{
+  { 0xBCBC3275, 0xECEC21F3, 0x202043C6, 0xB3B3C9F4, 0xDADA03DB, 0x02028B7B,
+    0xE2E22BFB, 0x9E9EFAC8, 0xC9C9EC4A, 0xD4D409D3, 0x18186BE6, 0x1E1E9F6B,
+    0x98980E45, 0xB2B2387D, 0xA6A6D2E8, 0x2626B74B, 0x3C3C57D6, 0x93938A32,
+    0x8282EED8, 0x525298FD, 0x7B7BD437, 0xBBBB3771, 0x5B5B97F1, 0x474783E1,
+    0x24243C30, 0x5151E20F, 0xBABAC6F8, 0x4A4AF31B, 0xBFBF4887, 0x0D0D70FA,
+    0xB0B0B306, 0x7575DE3F, 0xD2D2FD5E, 0x7D7D20BA, 0x666631AE, 0x3A3AA35B,
+    0x59591C8A, 0x00000000, 0xCDCD93BC, 0x1A1AE09D, 0xAEAE2C6D, 0x7F7FABC1,
+    0x2B2BC7B1, 0xBEBEB90E, 0xE0E0A080, 0x8A8A105D, 0x3B3B52D2, 0x6464BAD5,
+    0xD8D888A0, 0xE7E7A584, 0x5F5FE807, 0x1B1B1114, 0x2C2CC2B5, 0xFCFCB490,
+    0x3131272C, 0x808065A3, 0x73732AB2, 0x0C0C8173, 0x79795F4C, 0x6B6B4154,
+    0x4B4B0292, 0x53536974, 0x94948F36, 0x83831F51, 0x2A2A3638, 0xC4C49CB0,
+    0x2222C8BD, 0xD5D5F85A, 0xBDBDC3FC, 0x48487860, 0xFFFFCE62, 0x4C4C0796,
+    0x4141776C, 0xC7C7E642, 0xEBEB24F7, 0x1C1C1410, 0x5D5D637C, 0x36362228,
+    0x6767C027, 0xE9E9AF8C, 0x4444F913, 0x1414EA95, 0xF5F5BB9C, 0xCFCF18C7,
+    0x3F3F2D24, 0xC0C0E346, 0x7272DB3B, 0x54546C70, 0x29294CCA, 0xF0F035E3,
+    0x0808FE85, 0xC6C617CB, 0xF3F34F11, 0x8C8CE4D0, 0xA4A45993, 0xCACA96B8,
+    0x68683BA6, 0xB8B84D83, 0x38382820, 0xE5E52EFF, 0xADAD569F, 0x0B0B8477,
+    0xC8C81DC3, 0x9999FFCC, 0x5858ED03, 0x19199A6F, 0x0E0E0A08, 0x95957EBF,
+    0x70705040, 0xF7F730E7, 0x6E6ECF2B, 0x1F1F6EE2, 0xB5B53D79, 0x09090F0C,
+    0x616134AA, 0x57571682, 0x9F9F0B41, 0x9D9D803A, 0x111164EA, 0x2525CDB9,
+    0xAFAFDDE4, 0x4545089A, 0xDFDF8DA4, 0xA3A35C97, 0xEAEAD57E, 0x353558DA,
+    0xEDEDD07A, 0x4343FC17, 0xF8F8CB66, 0xFBFBB194, 0x3737D3A1, 0xFAFA401D,
+    0xC2C2683D, 0xB4B4CCF0, 0x32325DDE, 0x9C9C71B3, 0x5656E70B, 0xE3E3DA72,
+    0x878760A7, 0x15151B1C, 0xF9F93AEF, 0x6363BFD1, 0x3434A953, 0x9A9A853E,
+    0xB1B1428F, 0x7C7CD133, 0x88889B26, 0x3D3DA65F, 0xA1A1D7EC, 0xE4E4DF76,
+    0x8181942A, 0x91910149, 0x0F0FFB81, 0xEEEEAA88, 0x161661EE, 0xD7D77321,
+    0x9797F5C4, 0xA5A5A81A, 0xFEFE3FEB, 0x6D6DB5D9, 0x7878AEC5, 0xC5C56D39,
+    0x1D1DE599, 0x7676A4CD, 0x3E3EDCAD, 0xCBCB6731, 0xB6B6478B, 0xEFEF5B01,
+    0x12121E18, 0x6060C523, 0x6A6AB0DD, 0x4D4DF61F, 0xCECEE94E, 0xDEDE7C2D,
+    0x55559DF9, 0x7E7E5A48, 0x2121B24F, 0x03037AF2, 0xA0A02665, 0x5E5E198E,
+    0x5A5A6678, 0x65654B5C, 0x62624E58, 0xFDFD4519, 0x0606F48D, 0x404086E5,
+    0xF2F2BE98, 0x3333AC57, 0x17179067, 0x05058E7F, 0xE8E85E05, 0x4F4F7D64,
+    0x89896AAF, 0x10109563, 0x74742FB6, 0x0A0A75FE, 0x5C5C92F5, 0x9B9B74B7,
+    0x2D2D333C, 0x3030D6A5, 0x2E2E49CE, 0x494989E9, 0x46467268, 0x77775544,
+    0xA8A8D8E0, 0x9696044D, 0x2828BD43, 0xA9A92969, 0xD9D97929, 0x8686912E,
+    0xD1D187AC, 0xF4F44A15, 0x8D8D1559, 0xD6D682A8, 0xB9B9BC0A, 0x42420D9E,
+    0xF6F6C16E, 0x2F2FB847, 0xDDDD06DF, 0x23233934, 0xCCCC6235, 0xF1F1C46A,
+    0xC1C112CF, 0x8585EBDC, 0x8F8F9E22, 0x7171A1C9, 0x9090F0C0, 0xAAAA539B,
+    0x0101F189, 0x8B8BE1D4, 0x4E4E8CED, 0x8E8E6FAB, 0xABABA212, 0x6F6F3EA2,
+    0xE6E6540D, 0xDBDBF252, 0x92927BBB, 0xB7B7B602, 0x6969CA2F, 0x3939D9A9,
+    0xD3D30CD7, 0xA7A72361, 0xA2A2AD1E, 0xC3C399B4, 0x6C6C4450, 0x07070504,
+    0x04047FF6, 0x272746C2, 0xACACA716, 0xD0D07625, 0x50501386, 0xDCDCF756,
+    0x84841A55, 0xE1E15109, 0x7A7A25BE, 0x1313EF91 },
+
+  { 0xA9D93939, 0x67901717, 0xB3719C9C, 0xE8D2A6A6, 0x04050707, 0xFD985252,
+    0xA3658080, 0x76DFE4E4, 0x9A084545, 0x92024B4B, 0x80A0E0E0, 0x78665A5A,
+    0xE4DDAFAF, 0xDDB06A6A, 0xD1BF6363, 0x38362A2A, 0x0D54E6E6, 0xC6432020,
+    0x3562CCCC, 0x98BEF2F2, 0x181E1212, 0xF724EBEB, 0xECD7A1A1, 0x6C774141,
+    0x43BD2828, 0x7532BCBC, 0x37D47B7B, 0x269B8888, 0xFA700D0D, 0x13F94444,
+    0x94B1FBFB, 0x485A7E7E, 0xF27A0303, 0xD0E48C8C, 0x8B47B6B6, 0x303C2424,
+    0x84A5E7E7, 0x54416B6B, 0xDF06DDDD, 0x23C56060, 0x1945FDFD, 0x5BA33A3A,
+    0x3D68C2C2, 0x59158D8D, 0xF321ECEC, 0xAE316666, 0xA23E6F6F, 0x82165757,
+    0x63951010, 0x015BEFEF, 0x834DB8B8, 0x2E918686, 0xD9B56D6D, 0x511F8383,
+    0x9B53AAAA, 0x7C635D5D, 0xA63B6868, 0xEB3FFEFE, 0xA5D63030, 0xBE257A7A,
+    0x16A7ACAC, 0x0C0F0909, 0xE335F0F0, 0x6123A7A7, 0xC0F09090, 0x8CAFE9E9,
+    0x3A809D9D, 0xF5925C5C, 0x73810C0C, 0x2C273131, 0x2576D0D0, 0x0BE75656,
+    0xBB7B9292, 0x4EE9CECE, 0x89F10101, 0x6B9F1E1E, 0x53A93434, 0x6AC4F1F1,
+    0xB499C3C3, 0xF1975B5B, 0xE1834747, 0xE66B1818, 0xBDC82222, 0x450E9898,
+    0xE26E1F1F, 0xF4C9B3B3, 0xB62F7474, 0x66CBF8F8, 0xCCFF9999, 0x95EA1414,
+    0x03ED5858, 0x56F7DCDC, 0xD4E18B8B, 0x1C1B1515, 0x1EADA2A2, 0xD70CD3D3,
+    0xFB2BE2E2, 0xC31DC8C8, 0x8E195E5E, 0xB5C22C2C, 0xE9894949, 0xCF12C1C1,
+    0xBF7E9595, 0xBA207D7D, 0xEA641111, 0x77840B0B, 0x396DC5C5, 0xAF6A8989,
+    0x33D17C7C, 0xC9A17171, 0x62CEFFFF, 0x7137BBBB, 0x81FB0F0F, 0x793DB5B5,
+    0x0951E1E1, 0xADDC3E3E, 0x242D3F3F, 0xCDA47676, 0xF99D5555, 0xD8EE8282,
+    0xE5864040, 0xC5AE7878, 0xB9CD2525, 0x4D049696, 0x44557777, 0x080A0E0E,
+    0x86135050, 0xE730F7F7, 0xA1D33737, 0x1D40FAFA, 0xAA346161, 0xED8C4E4E,
+    0x06B3B0B0, 0x706C5454, 0xB22A7373, 0xD2523B3B, 0x410B9F9F, 0x7B8B0202,
+    0xA088D8D8, 0x114FF3F3, 0x3167CBCB, 0xC2462727, 0x27C06767, 0x90B4FCFC,
+    0x20283838, 0xF67F0404, 0x60784848, 0xFF2EE5E5, 0x96074C4C, 0x5C4B6565,
+    0xB1C72B2B, 0xAB6F8E8E, 0x9E0D4242, 0x9CBBF5F5, 0x52F2DBDB, 0x1BF34A4A,
+    0x5FA63D3D, 0x9359A4A4, 0x0ABCB9B9, 0xEF3AF9F9, 0x91EF1313, 0x85FE0808,
+    0x49019191, 0xEE611616, 0x2D7CDEDE, 0x4FB22121, 0x8F42B1B1, 0x3BDB7272,
+    0x47B82F2F, 0x8748BFBF, 0x6D2CAEAE, 0x46E3C0C0, 0xD6573C3C, 0x3E859A9A,
+    0x6929A9A9, 0x647D4F4F, 0x2A948181, 0xCE492E2E, 0xCB17C6C6, 0x2FCA6969,
+    0xFCC3BDBD, 0x975CA3A3, 0x055EE8E8, 0x7AD0EDED, 0xAC87D1D1, 0x7F8E0505,
+    0xD5BA6464, 0x1AA8A5A5, 0x4BB72626, 0x0EB9BEBE, 0xA7608787, 0x5AF8D5D5,
+    0x28223636, 0x14111B1B, 0x3FDE7575, 0x2979D9D9, 0x88AAEEEE, 0x3C332D2D,
+    0x4C5F7979, 0x02B6B7B7, 0xB896CACA, 0xDA583535, 0xB09CC4C4, 0x17FC4343,
+    0x551A8484, 0x1FF64D4D, 0x8A1C5959, 0x7D38B2B2, 0x57AC3333, 0xC718CFCF,
+    0x8DF40606, 0x74695353, 0xB7749B9B, 0xC4F59797, 0x9F56ADAD, 0x72DAE3E3,
+    0x7ED5EAEA, 0x154AF4F4, 0x229E8F8F, 0x12A2ABAB, 0x584E6262, 0x07E85F5F,
+    0x99E51D1D, 0x34392323, 0x6EC1F6F6, 0x50446C6C, 0xDE5D3232, 0x68724646,
+    0x6526A0A0, 0xBC93CDCD, 0xDB03DADA, 0xF8C6BABA, 0xC8FA9E9E, 0xA882D6D6,
+    0x2BCF6E6E, 0x40507070, 0xDCEB8585, 0xFE750A0A, 0x328A9393, 0xA48DDFDF,
+    0xCA4C2929, 0x10141C1C, 0x2173D7D7, 0xF0CCB4B4, 0xD309D4D4, 0x5D108A8A,
+    0x0FE25151, 0x00000000, 0x6F9A1919, 0x9DE01A1A, 0x368F9494, 0x42E6C7C7,
+    0x4AECC9C9, 0x5EFDD2D2, 0xC1AB7F7F, 0xE0D8A8A8 },
+
+  { 0xBC75BC32, 0xECF3EC21, 0x20C62043, 0xB3F4B3C9, 0xDADBDA03, 0x027B028B,
+    0xE2FBE22B, 0x9EC89EFA, 0xC94AC9EC, 0xD4D3D409, 0x18E6186B, 0x1E6B1E9F,
+    0x9845980E, 0xB27DB238, 0xA6E8A6D2, 0x264B26B7, 0x3CD63C57, 0x9332938A,
+    0x82D882EE, 0x52FD5298, 0x7B377BD4, 0xBB71BB37, 0x5BF15B97, 0x47E14783,
+    0x2430243C, 0x510F51E2, 0xBAF8BAC6, 0x4A1B4AF3, 0xBF87BF48, 0x0DFA0D70,
+    0xB006B0B3, 0x753F75DE, 0xD25ED2FD, 0x7DBA7D20, 0x66AE6631, 0x3A5B3AA3,
+    0x598A591C, 0x00000000, 0xCDBCCD93, 0x1A9D1AE0, 0xAE6DAE2C, 0x7FC17FAB,
+    0x2BB12BC7, 0xBE0EBEB9, 0xE080E0A0, 0x8A5D8A10, 0x3BD23B52, 0x64D564BA,
+    0xD8A0D888, 0xE784E7A5, 0x5F075FE8, 0x1B141B11, 0x2CB52CC2, 0xFC90FCB4,
+    0x312C3127, 0x80A38065, 0x73B2732A, 0x0C730C81, 0x794C795F, 0x6B546B41,
+    0x4B924B02, 0x53745369, 0x9436948F, 0x8351831F, 0x2A382A36, 0xC4B0C49C,
+    0x22BD22C8, 0xD55AD5F8, 0xBDFCBDC3, 0x48604878, 0xFF62FFCE, 0x4C964C07,
+    0x416C4177, 0xC742C7E6, 0xEBF7EB24, 0x1C101C14, 0x5D7C5D63, 0x36283622,
+    0x672767C0, 0xE98CE9AF, 0x441344F9, 0x149514EA, 0xF59CF5BB, 0xCFC7CF18,
+    0x3F243F2D, 0xC046C0E3, 0x723B72DB, 0x5470546C, 0x29CA294C, 0xF0E3F035,
+    0x088508FE, 0xC6CBC617, 0xF311F34F, 0x8CD08CE4, 0xA493A459, 0xCAB8CA96,
+    0x68A6683B, 0xB883B84D, 0x38203828, 0xE5FFE52E, 0xAD9FAD56, 0x0B770B84,
+    0xC8C3C81D, 0x99CC99FF, 0x580358ED, 0x196F199A, 0x0E080E0A, 0x95BF957E,
+    0x70407050, 0xF7E7F730, 0x6E2B6ECF, 0x1FE21F6E, 0xB579B53D, 0x090C090F,
+    0x61AA6134, 0x57825716, 0x9F419F0B, 0x9D3A9D80, 0x11EA1164, 0x25B925CD,
+    0xAFE4AFDD, 0x459A4508, 0xDFA4DF8D, 0xA397A35C, 0xEA7EEAD5, 0x35DA3558,
+    0xED7AEDD0, 0x431743FC, 0xF866F8CB, 0xFB94FBB1, 0x37A137D3, 0xFA1DFA40,
+    0xC23DC268, 0xB4F0B4CC, 0x32DE325D, 0x9CB39C71, 0x560B56E7, 0xE372E3DA,
+    0x87A78760, 0x151C151B, 0xF9EFF93A, 0x63D163BF, 0x345334A9, 0x9A3E9A85,
+    0xB18FB142, 0x7C337CD1, 0x8826889B, 0x3D5F3DA6, 0xA1ECA1D7, 0xE476E4DF,
+    0x812A8194, 0x91499101, 0x0F810FFB, 0xEE88EEAA, 0x16EE1661, 0xD721D773,
+    0x97C497F5, 0xA51AA5A8, 0xFEEBFE3F, 0x6DD96DB5, 0x78C578AE, 0xC539C56D,
+    0x1D991DE5, 0x76CD76A4, 0x3EAD3EDC, 0xCB31CB67, 0xB68BB647, 0xEF01EF5B,
+    0x1218121E, 0x602360C5, 0x6ADD6AB0, 0x4D1F4DF6, 0xCE4ECEE9, 0xDE2DDE7C,
+    0x55F9559D, 0x7E487E5A, 0x214F21B2, 0x03F2037A, 0xA065A026, 0x5E8E5E19,
+    0x5A785A66, 0x655C654B, 0x6258624E, 0xFD19FD45, 0x068D06F4, 0x40E54086,
+    0xF298F2BE, 0x335733AC, 0x17671790, 0x057F058E, 0xE805E85E, 0x4F644F7D,
+    0x89AF896A, 0x10631095, 0x74B6742F, 0x0AFE0A75, 0x5CF55C92, 0x9BB79B74,
+    0x2D3C2D33, 0x30A530D6, 0x2ECE2E49, 0x49E94989, 0x46684672, 0x77447755,
+    0xA8E0A8D8, 0x964D9604, 0x284328BD, 0xA969A929, 0xD929D979, 0x862E8691,
+    0xD1ACD187, 0xF415F44A, 0x8D598D15, 0xD6A8D682, 0xB90AB9BC, 0x429E420D,
+    0xF66EF6C1, 0x2F472FB8, 0xDDDFDD06, 0x23342339, 0xCC35CC62, 0xF16AF1C4,
+    0xC1CFC112, 0x85DC85EB, 0x8F228F9E, 0x71C971A1, 0x90C090F0, 0xAA9BAA53,
+    0x018901F1, 0x8BD48BE1, 0x4EED4E8C, 0x8EAB8E6F, 0xAB12ABA2, 0x6FA26F3E,
+    0xE60DE654, 0xDB52DBF2, 0x92BB927B, 0xB702B7B6, 0x692F69CA, 0x39A939D9,
+    0xD3D7D30C, 0xA761A723, 0xA21EA2AD, 0xC3B4C399, 0x6C506C44, 0x07040705,
+    0x04F6047F, 0x27C22746, 0xAC16ACA7, 0xD025D076, 0x50865013, 0xDC56DCF7,
+    0x8455841A, 0xE109E151, 0x7ABE7A25, 0x139113EF },
+
+  { 0xD939A9D9, 0x90176790, 0x719CB371, 0xD2A6E8D2, 0x05070405, 0x9852FD98,
+    0x6580A365, 0xDFE476DF, 0x08459A08, 0x024B9202, 0xA0E080A0, 0x665A7866,
+    0xDDAFE4DD, 0xB06ADDB0, 0xBF63D1BF, 0x362A3836, 0x54E60D54, 0x4320C643,
+    0x62CC3562, 0xBEF298BE, 0x1E12181E, 0x24EBF724, 0xD7A1ECD7, 0x77416C77,
+    0xBD2843BD, 0x32BC7532, 0xD47B37D4, 0x9B88269B, 0x700DFA70, 0xF94413F9,
+    0xB1FB94B1, 0x5A7E485A, 0x7A03F27A, 0xE48CD0E4, 0x47B68B47, 0x3C24303C,
+    0xA5E784A5, 0x416B5441, 0x06DDDF06, 0xC56023C5, 0x45FD1945, 0xA33A5BA3,
+    0x68C23D68, 0x158D5915, 0x21ECF321, 0x3166AE31, 0x3E6FA23E, 0x16578216,
+    0x95106395, 0x5BEF015B, 0x4DB8834D, 0x91862E91, 0xB56DD9B5, 0x1F83511F,
+    0x53AA9B53, 0x635D7C63, 0x3B68A63B, 0x3FFEEB3F, 0xD630A5D6, 0x257ABE25,
+    0xA7AC16A7, 0x0F090C0F, 0x35F0E335, 0x23A76123, 0xF090C0F0, 0xAFE98CAF,
+    0x809D3A80, 0x925CF592, 0x810C7381, 0x27312C27, 0x76D02576, 0xE7560BE7,
+    0x7B92BB7B, 0xE9CE4EE9, 0xF10189F1, 0x9F1E6B9F, 0xA93453A9, 0xC4F16AC4,
+    0x99C3B499, 0x975BF197, 0x8347E183, 0x6B18E66B, 0xC822BDC8, 0x0E98450E,
+    0x6E1FE26E, 0xC9B3F4C9, 0x2F74B62F, 0xCBF866CB, 0xFF99CCFF, 0xEA1495EA,
+    0xED5803ED, 0xF7DC56F7, 0xE18BD4E1, 0x1B151C1B, 0xADA21EAD, 0x0CD3D70C,
+    0x2BE2FB2B, 0x1DC8C31D, 0x195E8E19, 0xC22CB5C2, 0x8949E989, 0x12C1CF12,
+    0x7E95BF7E, 0x207DBA20, 0x6411EA64, 0x840B7784, 0x6DC5396D, 0x6A89AF6A,
+    0xD17C33D1, 0xA171C9A1, 0xCEFF62CE, 0x37BB7137, 0xFB0F81FB, 0x3DB5793D,
+    0x51E10951, 0xDC3EADDC, 0x2D3F242D, 0xA476CDA4, 0x9D55F99D, 0xEE82D8EE,
+    0x8640E586, 0xAE78C5AE, 0xCD25B9CD, 0x04964D04, 0x55774455, 0x0A0E080A,
+    0x13508613, 0x30F7E730, 0xD337A1D3, 0x40FA1D40, 0x3461AA34, 0x8C4EED8C,
+    0xB3B006B3, 0x6C54706C, 0x2A73B22A, 0x523BD252, 0x0B9F410B, 0x8B027B8B,
+    0x88D8A088, 0x4FF3114F, 0x67CB3167, 0x4627C246, 0xC06727C0, 0xB4FC90B4,
+    0x28382028, 0x7F04F67F, 0x78486078, 0x2EE5FF2E, 0x074C9607, 0x4B655C4B,
+    0xC72BB1C7, 0x6F8EAB6F, 0x0D429E0D, 0xBBF59CBB, 0xF2DB52F2, 0xF34A1BF3,
+    0xA63D5FA6, 0x59A49359, 0xBCB90ABC, 0x3AF9EF3A, 0xEF1391EF, 0xFE0885FE,
+    0x01914901, 0x6116EE61, 0x7CDE2D7C, 0xB2214FB2, 0x42B18F42, 0xDB723BDB,
+    0xB82F47B8, 0x48BF8748, 0x2CAE6D2C, 0xE3C046E3, 0x573CD657, 0x859A3E85,
+    0x29A96929, 0x7D4F647D, 0x94812A94, 0x492ECE49, 0x17C6CB17, 0xCA692FCA,
+    0xC3BDFCC3, 0x5CA3975C, 0x5EE8055E, 0xD0ED7AD0, 0x87D1AC87, 0x8E057F8E,
+    0xBA64D5BA, 0xA8A51AA8, 0xB7264BB7, 0xB9BE0EB9, 0x6087A760, 0xF8D55AF8,
+    0x22362822, 0x111B1411, 0xDE753FDE, 0x79D92979, 0xAAEE88AA, 0x332D3C33,
+    0x5F794C5F, 0xB6B702B6, 0x96CAB896, 0x5835DA58, 0x9CC4B09C, 0xFC4317FC,
+    0x1A84551A, 0xF64D1FF6, 0x1C598A1C, 0x38B27D38, 0xAC3357AC, 0x18CFC718,
+    0xF4068DF4, 0x69537469, 0x749BB774, 0xF597C4F5, 0x56AD9F56, 0xDAE372DA,
+    0xD5EA7ED5, 0x4AF4154A, 0x9E8F229E, 0xA2AB12A2, 0x4E62584E, 0xE85F07E8,
+    0xE51D99E5, 0x39233439, 0xC1F66EC1, 0x446C5044, 0x5D32DE5D, 0x72466872,
+    0x26A06526, 0x93CDBC93, 0x03DADB03, 0xC6BAF8C6, 0xFA9EC8FA, 0x82D6A882,
+    0xCF6E2BCF, 0x50704050, 0xEB85DCEB, 0x750AFE75, 0x8A93328A, 0x8DDFA48D,
+    0x4C29CA4C, 0x141C1014, 0x73D72173, 0xCCB4F0CC, 0x09D4D309, 0x108A5D10,
+    0xE2510FE2, 0x00000000, 0x9A196F9A, 0xE01A9DE0, 0x8F94368F, 0xE6C742E6,
+    0xECC94AEC, 0xFDD25EFD, 0xAB7FC1AB, 0xD8A8E0D8 }
+};
+
+#define mds(n,x) m_tab[n][x]
+
+u32 h_fun (u32 *sk, u32 *lk, const u32 x, const u32 *key)
+{
+  u32  b0, b1, b2, b3;
+
+  b0 = extract_byte (x, 0);
+  b1 = extract_byte (x, 1);
+  b2 = extract_byte (x, 2);
+  b3 = extract_byte (x, 3);
+
+  b0 = q (1, b0) ^ extract_byte (key[3], 0);
+  b1 = q (0, b1) ^ extract_byte (key[3], 1);
+  b2 = q (0, b2) ^ extract_byte (key[3], 2);
+  b3 = q (1, b3) ^ extract_byte (key[3], 3);
+
+  b0 = q (1, b0) ^ extract_byte (key[2], 0);
+  b1 = q (1, b1) ^ extract_byte (key[2], 1);
+  b2 = q (0, b2) ^ extract_byte (key[2], 2);
+  b3 = q (0, b3) ^ extract_byte (key[2], 3);
+
+  b0 = q (0, (q (0, b0) ^ extract_byte (key[1], 0))) ^ extract_byte (key[0], 0);
+  b1 = q (0, (q (1, b1) ^ extract_byte (key[1], 1))) ^ extract_byte (key[0], 1);
+  b2 = q (1, (q (0, b2) ^ extract_byte (key[1], 2))) ^ extract_byte (key[0], 2);
+  b3 = q (1, (q (1, b3) ^ extract_byte (key[1], 3))) ^ extract_byte (key[0], 3);
+
+  return mds (0, b0) ^ mds (1, b1) ^ mds (2, b2) ^ mds (3, b3);
+}
+
+#define q40(x,k) q (0, q (0, q (1, q (1, x) ^ extract_byte (k[3], 0)) ^ extract_byte (k[2], 0)) ^ extract_byte (k[1], 0)) ^ extract_byte (k[0], 0)
+#define q41(x,k) q (0, q (1, q (1, q (0, x) ^ extract_byte (k[3], 1)) ^ extract_byte (k[2], 1)) ^ extract_byte (k[1], 1)) ^ extract_byte (k[0], 1)
+#define q42(x,k) q (1, q (0, q (0, q (0, x) ^ extract_byte (k[3], 2)) ^ extract_byte (k[2], 2)) ^ extract_byte (k[1], 2)) ^ extract_byte (k[0], 2)
+#define q43(x,k) q (1, q (1, q (0, q (1, x) ^ extract_byte (k[3], 3)) ^ extract_byte (k[2], 3)) ^ extract_byte (k[1], 3)) ^ extract_byte (k[0], 3)
+
+#define g1_fun(x)                                   \
+  (mds (0, q40 (extract_byte (x, 3), sk)) ^ \
+   mds (1, q41 (extract_byte (x, 0), sk)) ^ \
+   mds (2, q42 (extract_byte (x, 1), sk)) ^ \
+   mds (3, q43 (extract_byte (x, 2), sk)))
+
+#define g0_fun(x)                                   \
+  (mds (0, q40 (extract_byte (x, 0), sk)) ^ \
+   mds (1, q41 (extract_byte (x, 1), sk)) ^ \
+   mds (2, q42 (extract_byte (x, 2), sk)) ^ \
+   mds (3, q43 (extract_byte (x, 3), sk)))
+
+u32 mds_rem (u32 p0, u32 p1)
+{
+  #define G_MOD 0x14d
+
+  for (int i = 0; i < 8; i++)
+  {
+    u32 t = p1 >> 24;
+
+    p1 = (p1 << 8) | (p0 >> 24);
+
+    p0 <<= 8;
+
+    u32 u = (t << 1);
+
+    if (t & 0x80) u ^= G_MOD;
+
+    p1 ^= t ^ (u << 16);
+
+    u ^= (t >> 1);
+
+    if (t & 0x01) u ^= G_MOD >> 1;
+
+    p1 ^= (u << 24) | (u << 8);
+  }
+
+  return p1;
+}
+
+void twofish256_set_key (u32 *sk, u32 *lk, const u32 *ukey)
+{
+  u32 me_key[4];
+
+  me_key[0] = ukey[0];
+  me_key[1] = ukey[2];
+  me_key[2] = ukey[4];
+  me_key[3] = ukey[6];
+
+  u32 mo_key[4];
+
+  mo_key[0] = ukey[1];
+  mo_key[1] = ukey[3];
+  mo_key[2] = ukey[5];
+  mo_key[3] = ukey[7];
+
+  sk[3] = mds_rem (me_key[0], mo_key[0]);
+  sk[2] = mds_rem (me_key[1], mo_key[1]);
+  sk[1] = mds_rem (me_key[2], mo_key[2]);
+  sk[0] = mds_rem (me_key[3], mo_key[3]);
+
+  for (int i = 0; i < 40; i += 2)
+  {
+    u32 a = 0x01010101 * i;
+    u32 b = 0x01010101 + a;
+
+    a = h_fun (sk, lk, a, me_key);
+    b = h_fun (sk, lk, b, mo_key);
+
+    b = rotl32 (b, 8);
+
+    lk[i + 0] = a + b;
+    lk[i + 1] = rotl32 (a + 2 * b, 9);
+  }
+}
+
+#define f_rnd(i)                                                    \
+{                                                                   \
+  u32 t0 = g0_fun (data[0]);                                       \
+  u32 t1 = g1_fun (data[1]);                                       \
+  data[2] = rotr32 (data[2] ^ (t0 + t1 + lk[4 * (i) + 8]), 1);      \
+  data[3] = rotl32 (data[3], 1) ^ (t0 + 2 * t1 + lk[4 * (i) + 9]);  \
+  u32 t2 = g0_fun (data[2]);                                       \
+  u32 t3 = g1_fun (data[3]);                                       \
+  data[0] = rotr32 (data[0] ^ (t2 + t3 + lk[4 * (i) + 10]), 1);     \
+  data[1] = rotl32 (data[1], 1) ^ (t2 + 2 * t3 + lk[4 * (i) + 11]); \
+}
+
+void twofish256_encrypt (const u32 *sk, const u32 *lk, const u32 *in, u32 *out)
+{
+  u32 data[4];
+
+  data[0] = in[0] ^ lk[0];
+  data[1] = in[1] ^ lk[1];
+  data[2] = in[2] ^ lk[2];
+  data[3] = in[3] ^ lk[3];
+
+  f_rnd (0);
+  f_rnd (1);
+  f_rnd (2);
+  f_rnd (3);
+  f_rnd (4);
+  f_rnd (5);
+  f_rnd (6);
+  f_rnd (7);
+
+  out[0] = data[2] ^ lk[4];
+  out[1] = data[3] ^ lk[5];
+  out[2] = data[0] ^ lk[6];
+  out[3] = data[1] ^ lk[7];
+}
+
+#define i_rnd(i)                                                    \
+{                                                                   \
+  u32 t0 = g0_fun (data[0]);                                       \
+  u32 t1 = g1_fun (data[1]);                                       \
+  data[2] = rotl32 (data[2], 1) ^ (t0 + t1 + lk[4 * (i) + 10]);     \
+  data[3] = rotr32 (data[3] ^ (t0 + 2 * t1 + lk[4 * (i) + 11]), 1); \
+  u32 t2 = g0_fun (data[2]);                                       \
+  u32 t3 = g1_fun (data[3]);                                       \
+  data[0] = rotl32 (data[0], 1) ^ (t2 + t3 + lk[4 * (i) +  8]);     \
+  data[1] = rotr32 (data[1] ^ (t2 + 2 * t3 + lk[4 * (i) +  9]), 1); \
+}
+
+void twofish256_decrypt (const u32 *sk, const u32 *lk, const u32 *in, u32 *out)
+{
+  u32 data[4];
+
+  data[0] = in[0] ^ lk[4];
+  data[1] = in[1] ^ lk[5];
+  data[2] = in[2] ^ lk[6];
+  data[3] = in[3] ^ lk[7];
+
+  i_rnd (7);
+  i_rnd (6);
+  i_rnd (5);
+  i_rnd (4);
+  i_rnd (3);
+  i_rnd (2);
+  i_rnd (1);
+  i_rnd (0);
+
+  out[0] = data[2] ^ lk[0];
+  out[1] = data[3] ^ lk[1];
+  out[2] = data[0] ^ lk[2];
+  out[3] = data[1] ^ lk[3];
+}
+
+void twofish256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out)
+{
+  u32 T[4] = { 0 };
+  u32 Z[4] = { 0 };
+
+  out[0] = in[0];
+  out[1] = in[1];
+  out[2] = in[2];
+  out[3] = in[3];
+
+  u32 sk[4]; u32 lk[40];
+
+  twofish256_set_key (sk, lk, ukey2);
+  twofish256_encrypt (sk, lk, Z, T);
+
+  out[0] ^= T[0];
+  out[1] ^= T[1];
+  out[2] ^= T[2];
+  out[3] ^= T[3];
+
+  twofish256_set_key (sk, lk, ukey1);
+  twofish256_decrypt (sk, lk, out, out);
+
+  out[0] ^= T[0];
+  out[1] ^= T[1];
+  out[2] ^= T[2];
+  out[3] ^= T[3];
+}

diff --git a/OpenCL/inc_common.cl b/OpenCL/inc_common.cl
new file mode 100644 (file)
index 0000000..ca811e5
--- /dev/null
+++ b/OpenCL/inc_common.cl
@@ -0,0 +1,9324 @@
+/**
+ * Author......: Jens Steube <jens.steube@gmail.com>
+ * License.....: MIT
+ */
+
+/**
+ * pure scalar functions
+ */
+
+inline int hash_comp (const u32 d1[4], __global u32 *d2)
+{
+  if (d1[3] > d2[DGST_R3]) return ( 1);
+  if (d1[3] < d2[DGST_R3]) return (-1);
+  if (d1[2] > d2[DGST_R2]) return ( 1);
+  if (d1[2] < d2[DGST_R2]) return (-1);
+  if (d1[1] > d2[DGST_R1]) return ( 1);
+  if (d1[1] < d2[DGST_R1]) return (-1);
+  if (d1[0] > d2[DGST_R0]) return ( 1);
+  if (d1[0] < d2[DGST_R0]) return (-1);
+
+  return (0);
+}
+
+inline int find_hash (const u32 digest[4], const u32 digests_cnt, __global digest_t *digests_buf)
+{
+  for (u32 l = 0, r = digests_cnt; r; r >>= 1)
+  {
+    const u32 m = r >> 1;
+
+    const u32 c = l + m;
+
+    const int cmp = hash_comp (digest, digests_buf[c].digest_buf);
+
+    if (cmp > 0)
+    {
+      l += m + 1;
+
+      r--;
+    }
+
+    if (cmp == 0) return (c);
+  }
+
+  return (-1);
+}
+
+inline u32 check_bitmap (__global u32 *bitmap, const u32 bitmap_mask, const u32 bitmap_shift, const u32 digest)
+{
+  return (bitmap[(digest >> bitmap_shift) & bitmap_mask] & (1 << (digest & 0x1f)));
+}
+
+inline u32 check (const u32 digest[2], __global u32 *bitmap_s1_a, __global u32 *bitmap_s1_b, __global u32 *bitmap_s1_c, __global u32 *bitmap_s1_d, __global u32 *bitmap_s2_a, __global u32 *bitmap_s2_b, __global u32 *bitmap_s2_c, __global u32 *bitmap_s2_d, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2)
+{
+  if (check_bitmap (bitmap_s1_a, bitmap_mask, bitmap_shift1, digest[0]) == 0) return (0);
+  if (check_bitmap (bitmap_s1_b, bitmap_mask, bitmap_shift1, digest[1]) == 0) return (0);
+  if (check_bitmap (bitmap_s1_c, bitmap_mask, bitmap_shift1, digest[2]) == 0) return (0);
+  if (check_bitmap (bitmap_s1_d, bitmap_mask, bitmap_shift1, digest[3]) == 0) return (0);
+
+  if (check_bitmap (bitmap_s2_a, bitmap_mask, bitmap_shift2, digest[0]) == 0) return (0);
+  if (check_bitmap (bitmap_s2_b, bitmap_mask, bitmap_shift2, digest[1]) == 0) return (0);
+  if (check_bitmap (bitmap_s2_c, bitmap_mask, bitmap_shift2, digest[2]) == 0) return (0);
+  if (check_bitmap (bitmap_s2_d, bitmap_mask, bitmap_shift2, digest[3]) == 0) return (0);
+
+  return (1);
+}
+
+inline void mark_hash (__global plain_t *plains_buf, __global u32 *d_result, const int salt_pos, const int digest_pos, const int hash_pos, const u32 gid, const u32 il_pos)
+{
+  const u32 idx = atomic_inc (d_result);
+
+  plains_buf[idx].salt_pos    = salt_pos;
+  plains_buf[idx].digest_pos  = digest_pos; // relative
+  plains_buf[idx].hash_pos    = hash_pos;   // absolute
+  plains_buf[idx].gidvid      = gid;
+  plains_buf[idx].il_pos      = il_pos;
+}
+
+/**
+ * vector functions
+ */
+
+inline void truncate_block (u32x w[4], const u32 len)
+{
+  switch (len)
+  {
+    case  0:  w[0] &= 0;
+              w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  1:  w[0] &= 0x000000FF;
+              w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  2:  w[0] &= 0x0000FFFF;
+              w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  3:  w[0] &= 0x00FFFFFF;
+              w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  4:  w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  5:  w[1] &= 0x000000FF;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  6:  w[1] &= 0x0000FFFF;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  7:  w[1] &= 0x00FFFFFF;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  8:  w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  9:  w[2] &= 0x000000FF;
+              w[3] &= 0;
+              break;
+    case 10:  w[2] &= 0x0000FFFF;
+              w[3] &= 0;
+              break;
+    case 11:  w[2] &= 0x00FFFFFF;
+              w[3] &= 0;
+              break;
+    case 12:  w[3] &= 0;
+              break;
+    case 13:  w[3] &= 0x000000FF;
+              break;
+    case 14:  w[3] &= 0x0000FFFF;
+              break;
+    case 15:  w[3] &= 0x00FFFFFF;
+              break;
+  }
+}
+
+inline void make_unicode (const u32x in[4], u32x out1[4], u32x out2[4])
+{
+  #ifdef IS_NV
+  out2[3] = __byte_perm (in[3], 0, 0x7372);
+  out2[2] = __byte_perm (in[3], 0, 0x7170);
+  out2[1] = __byte_perm (in[2], 0, 0x7372);
+  out2[0] = __byte_perm (in[2], 0, 0x7170);
+  out1[3] = __byte_perm (in[1], 0, 0x7372);
+  out1[2] = __byte_perm (in[1], 0, 0x7170);
+  out1[1] = __byte_perm (in[0], 0, 0x7372);
+  out1[0] = __byte_perm (in[0], 0, 0x7170);
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  out2[3]  = ((in[3] >> 8) & 0x00FF0000) | ((in[3] >> 16) & 0x000000FF);
+  out2[2]  = ((in[3] << 8) & 0x00FF0000) | ((in[3] >>  0) & 0x000000FF);
+  out2[1]  = ((in[2] >> 8) & 0x00FF0000) | ((in[2] >> 16) & 0x000000FF);
+  out2[0]  = ((in[2] << 8) & 0x00FF0000) | ((in[2] >>  0) & 0x000000FF);
+  out1[3]  = ((in[1] >> 8) & 0x00FF0000) | ((in[1] >> 16) & 0x000000FF);
+  out1[2]  = ((in[1] << 8) & 0x00FF0000) | ((in[1] >>  0) & 0x000000FF);
+  out1[1]  = ((in[0] >> 8) & 0x00FF0000) | ((in[0] >> 16) & 0x000000FF);
+  out1[0]  = ((in[0] << 8) & 0x00FF0000) | ((in[0] >>  0) & 0x000000FF);
+  #endif
+}
+
+inline void undo_unicode (const u32x in1[4], const u32x in2[4], u32x out[4])
+{
+  #ifdef IS_NV
+  out[0] = __byte_perm (in1[0], in1[1], 0x6420);
+  out[1] = __byte_perm (in1[2], in1[3], 0x6420);
+  out[2] = __byte_perm (in2[0], in2[1], 0x6420);
+  out[3] = __byte_perm (in2[2], in2[3], 0x6420);
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  out[0] = ((in1[0] & 0x000000ff) >>  0) | ((in1[0] & 0x00ff0000) >>  8)
+         | ((in1[1] & 0x000000ff) << 16) | ((in1[1] & 0x00ff0000) <<  8);
+  out[1] = ((in1[2] & 0x000000ff) >>  0) | ((in1[2] & 0x00ff0000) >>  8)
+         | ((in1[3] & 0x000000ff) << 16) | ((in1[3] & 0x00ff0000) <<  8);
+  out[2] = ((in2[0] & 0x000000ff) >>  0) | ((in2[0] & 0x00ff0000) >>  8)
+         | ((in2[1] & 0x000000ff) << 16) | ((in2[1] & 0x00ff0000) <<  8);
+  out[3] = ((in2[2] & 0x000000ff) >>  0) | ((in2[2] & 0x00ff0000) >>  8)
+         | ((in2[3] & 0x000000ff) << 16) | ((in2[3] & 0x00ff0000) <<  8);
+  #endif
+}
+
+inline void append_0x01_1x4 (u32x w0[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0]  = 0x01;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0100;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x010000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x01000000;
+      break;
+
+    case 4:
+      w0[1] = 0x01;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0100;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x010000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x01000000;
+      break;
+
+    case 8:
+      w0[2] = 0x01;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0100;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x010000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x01000000;
+      break;
+
+    case 12:
+      w0[3] = 0x01;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0100;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x010000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x01000000;
+      break;
+  }
+}
+
+inline void append_0x01_2x4 (u32x w0[4], u32x w1[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x01;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0100;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x010000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x01000000;
+      break;
+
+    case 4:
+      w0[1] = 0x01;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0100;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x010000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x01000000;
+      break;
+
+    case 8:
+      w0[2] = 0x01;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0100;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x010000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x01000000;
+      break;
+
+    case 12:
+      w0[3] = 0x01;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0100;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x010000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x01000000;
+      break;
+
+    case 16:
+      w1[0] = 0x01;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0100;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x010000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x01000000;
+      break;
+
+    case 20:
+      w1[1] = 0x01;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0100;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x010000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x01000000;
+      break;
+
+    case 24:
+      w1[2] = 0x01;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0100;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x010000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x01000000;
+      break;
+
+    case 28:
+      w1[3] = 0x01;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0100;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x010000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x01000000;
+      break;
+  }
+}
+
+inline void append_0x01_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x01;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0100;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x010000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x01000000;
+      break;
+
+    case 4:
+      w0[1] = 0x01;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0100;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x010000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x01000000;
+      break;
+
+    case 8:
+      w0[2] = 0x01;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0100;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x010000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x01000000;
+      break;
+
+    case 12:
+      w0[3] = 0x01;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0100;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x010000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x01000000;
+      break;
+
+    case 16:
+      w1[0] = 0x01;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0100;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x010000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x01000000;
+      break;
+
+    case 20:
+      w1[1] = 0x01;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0100;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x010000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x01000000;
+      break;
+
+    case 24:
+      w1[2] = 0x01;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0100;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x010000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x01000000;
+      break;
+
+    case 28:
+      w1[3] = 0x01;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0100;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x010000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x01000000;
+      break;
+
+    case 32:
+      w2[0] = 0x01;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x0100;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x010000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x01000000;
+      break;
+
+    case 36:
+      w2[1] = 0x01;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x0100;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x010000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x01000000;
+      break;
+
+    case 40:
+      w2[2] = 0x01;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x0100;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x010000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x01000000;
+      break;
+
+    case 44:
+      w2[3] = 0x01;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x0100;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x010000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x01000000;
+      break;
+  }
+}
+
+inline void append_0x01_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x01;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0100;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x010000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x01000000;
+      break;
+
+    case 4:
+      w0[1] = 0x01;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0100;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x010000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x01000000;
+      break;
+
+    case 8:
+      w0[2] = 0x01;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0100;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x010000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x01000000;
+      break;
+
+    case 12:
+      w0[3] = 0x01;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0100;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x010000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x01000000;
+      break;
+
+    case 16:
+      w1[0] = 0x01;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0100;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x010000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x01000000;
+      break;
+
+    case 20:
+      w1[1] = 0x01;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0100;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x010000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x01000000;
+      break;
+
+    case 24:
+      w1[2] = 0x01;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0100;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x010000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x01000000;
+      break;
+
+    case 28:
+      w1[3] = 0x01;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0100;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x010000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x01000000;
+      break;
+
+    case 32:
+      w2[0] = 0x01;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x0100;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x010000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x01000000;
+      break;
+
+    case 36:
+      w2[1] = 0x01;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x0100;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x010000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x01000000;
+      break;
+
+    case 40:
+      w2[2] = 0x01;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x0100;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x010000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x01000000;
+      break;
+
+    case 44:
+      w2[3] = 0x01;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x0100;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x010000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x01000000;
+      break;
+
+    case 48:
+      w3[0] = 0x01;
+      break;
+
+    case 49:
+      w3[0] = w3[0] | 0x0100;
+      break;
+
+    case 50:
+      w3[0] = w3[0] | 0x010000;
+      break;
+
+    case 51:
+      w3[0] = w3[0] | 0x01000000;
+      break;
+
+    case 52:
+      w3[1] = 0x01;
+      break;
+
+    case 53:
+      w3[1] = w3[1] | 0x0100;
+      break;
+
+    case 54:
+      w3[1] = w3[1] | 0x010000;
+      break;
+
+    case 55:
+      w3[1] = w3[1] | 0x01000000;
+      break;
+
+    case 56:
+      w3[2] = 0x01;
+      break;
+
+    case 57:
+      w3[2] = w3[2] | 0x0100;
+      break;
+
+    case 58:
+      w3[2] = w3[2] | 0x010000;
+      break;
+
+    case 59:
+      w3[2] = w3[2] | 0x01000000;
+      break;
+
+    case 60:
+      w3[3] = 0x01;
+      break;
+
+    case 61:
+      w3[3] = w3[3] | 0x0100;
+      break;
+
+    case 62:
+      w3[3] = w3[3] | 0x010000;
+      break;
+
+    case 63:
+      w3[3] = w3[3] | 0x01000000;
+      break;
+  }
+}
+
+inline void append_0x01_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x01;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0100;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x010000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x01000000;
+      break;
+
+    case 4:
+      w0[1] = 0x01;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0100;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x010000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x01000000;
+      break;
+
+    case 8:
+      w0[2] = 0x01;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0100;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x010000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x01000000;
+      break;
+
+    case 12:
+      w0[3] = 0x01;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0100;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x010000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x01000000;
+      break;
+
+    case 16:
+      w1[0] = 0x01;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0100;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x010000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x01000000;
+      break;
+
+    case 20:
+      w1[1] = 0x01;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0100;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x010000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x01000000;
+      break;
+
+    case 24:
+      w1[2] = 0x01;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0100;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x010000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x01000000;
+      break;
+
+    case 28:
+      w1[3] = 0x01;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0100;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x010000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x01000000;
+      break;
+
+    case 32:
+      w2[0] = 0x01;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x0100;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x010000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x01000000;
+      break;
+
+    case 36:
+      w2[1] = 0x01;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x0100;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x010000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x01000000;
+      break;
+
+    case 40:
+      w2[2] = 0x01;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x0100;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x010000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x01000000;
+      break;
+
+    case 44:
+      w2[3] = 0x01;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x0100;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x010000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x01000000;
+      break;
+
+    case 48:
+      w3[0] = 0x01;
+      break;
+
+    case 49:
+      w3[0] = w3[0] | 0x0100;
+      break;
+
+    case 50:
+      w3[0] = w3[0] | 0x010000;
+      break;
+
+    case 51:
+      w3[0] = w3[0] | 0x01000000;
+      break;
+
+    case 52:
+      w3[1] = 0x01;
+      break;
+
+    case 53:
+      w3[1] = w3[1] | 0x0100;
+      break;
+
+    case 54:
+      w3[1] = w3[1] | 0x010000;
+      break;
+
+    case 55:
+      w3[1] = w3[1] | 0x01000000;
+      break;
+
+    case 56:
+      w3[2] = 0x01;
+      break;
+
+    case 57:
+      w3[2] = w3[2] | 0x0100;
+      break;
+
+    case 58:
+      w3[2] = w3[2] | 0x010000;
+      break;
+
+    case 59:
+      w3[2] = w3[2] | 0x01000000;
+      break;
+
+    case 60:
+      w3[3] = 0x01;
+      break;
+
+    case 61:
+      w3[3] = w3[3] | 0x0100;
+      break;
+
+    case 62:
+      w3[3] = w3[3] | 0x010000;
+      break;
+
+    case 63:
+      w3[3] = w3[3] | 0x01000000;
+      break;
+
+    case 64:
+      w4[0] = 0x01;
+      break;
+
+    case 65:
+      w4[0] = w4[0] | 0x0100;
+      break;
+
+    case 66:
+      w4[0] = w4[0] | 0x010000;
+      break;
+
+    case 67:
+      w4[0] = w4[0] | 0x01000000;
+      break;
+
+    case 68:
+      w4[1] = 0x01;
+      break;
+
+    case 69:
+      w4[1] = w4[1] | 0x0100;
+      break;
+
+    case 70:
+      w4[1] = w4[1] | 0x010000;
+      break;
+
+    case 71:
+      w4[1] = w4[1] | 0x01000000;
+      break;
+
+    case 72:
+      w4[2] = 0x01;
+      break;
+
+    case 73:
+      w4[2] = w4[2] | 0x0100;
+      break;
+
+    case 74:
+      w4[2] = w4[2] | 0x010000;
+      break;
+
+    case 75:
+      w4[2] = w4[2] | 0x01000000;
+      break;
+
+    case 76:
+      w4[3] = 0x01;
+      break;
+
+    case 77:
+      w4[3] = w4[3] | 0x0100;
+      break;
+
+    case 78:
+      w4[3] = w4[3] | 0x010000;
+      break;
+
+    case 79:
+      w4[3] = w4[3] | 0x01000000;
+      break;
+
+    case 80:
+      w5[0] = 0x01;
+      break;
+
+    case 81:
+      w5[0] = w5[0] | 0x0100;
+      break;
+
+    case 82:
+      w5[0] = w5[0] | 0x010000;
+      break;
+
+    case 83:
+      w5[0] = w5[0] | 0x01000000;
+      break;
+
+    case 84:
+      w5[1] = 0x01;
+      break;
+
+    case 85:
+      w5[1] = w5[1] | 0x0100;
+      break;
+
+    case 86:
+      w5[1] = w5[1] | 0x010000;
+      break;
+
+    case 87:
+      w5[1] = w5[1] | 0x01000000;
+      break;
+
+    case 88:
+      w5[2] = 0x01;
+      break;
+
+    case 89:
+      w5[2] = w5[2] | 0x0100;
+      break;
+
+    case 90:
+      w5[2] = w5[2] | 0x010000;
+      break;
+
+    case 91:
+      w5[2] = w5[2] | 0x01000000;
+      break;
+
+    case 92:
+      w5[3] = 0x01;
+      break;
+
+    case 93:
+      w5[3] = w5[3] | 0x0100;
+      break;
+
+    case 94:
+      w5[3] = w5[3] | 0x010000;
+      break;
+
+    case 95:
+      w5[3] = w5[3] | 0x01000000;
+      break;
+
+    case 96:
+      w6[0] = 0x01;
+      break;
+
+    case 97:
+      w6[0] = w6[0] | 0x0100;
+      break;
+
+    case 98:
+      w6[0] = w6[0] | 0x010000;
+      break;
+
+    case 99:
+      w6[0] = w6[0] | 0x01000000;
+      break;
+
+    case 100:
+      w6[1] = 0x01;
+      break;
+
+    case 101:
+      w6[1] = w6[1] | 0x0100;
+      break;
+
+    case 102:
+      w6[1] = w6[1] | 0x010000;
+      break;
+
+    case 103:
+      w6[1] = w6[1] | 0x01000000;
+      break;
+
+    case 104:
+      w6[2] = 0x01;
+      break;
+
+    case 105:
+      w6[2] = w6[2] | 0x0100;
+      break;
+
+    case 106:
+      w6[2] = w6[2] | 0x010000;
+      break;
+
+    case 107:
+      w6[2] = w6[2] | 0x01000000;
+      break;
+
+    case 108:
+      w6[3] = 0x01;
+      break;
+
+    case 109:
+      w6[3] = w6[3] | 0x0100;
+      break;
+
+    case 110:
+      w6[3] = w6[3] | 0x010000;
+      break;
+
+    case 111:
+      w6[3] = w6[3] | 0x01000000;
+      break;
+
+    case 112:
+      w7[0] = 0x01;
+      break;
+
+    case 113:
+      w7[0] = w7[0] | 0x0100;
+      break;
+
+    case 114:
+      w7[0] = w7[0] | 0x010000;
+      break;
+
+    case 115:
+      w7[0] = w7[0] | 0x01000000;
+      break;
+
+    case 116:
+      w7[1] = 0x01;
+      break;
+
+    case 117:
+      w7[1] = w7[1] | 0x0100;
+      break;
+
+    case 118:
+      w7[1] = w7[1] | 0x010000;
+      break;
+
+    case 119:
+      w7[1] = w7[1] | 0x01000000;
+      break;
+
+    case 120:
+      w7[2] = 0x01;
+      break;
+
+    case 121:
+      w7[2] = w7[2] | 0x0100;
+      break;
+
+    case 122:
+      w7[2] = w7[2] | 0x010000;
+      break;
+
+    case 123:
+      w7[2] = w7[2] | 0x01000000;
+      break;
+
+    case 124:
+      w7[3] = 0x01;
+      break;
+
+    case 125:
+      w7[3] = w7[3] | 0x0100;
+      break;
+
+    case 126:
+      w7[3] = w7[3] | 0x010000;
+      break;
+
+    case 127:
+      w7[3] = w7[3] | 0x01000000;
+      break;
+  }
+}
+
+inline void append_0x02_1x4 (u32x w0[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0]  = 0x02;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0200;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x020000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x02000000;
+      break;
+
+    case 4:
+      w0[1] = 0x02;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0200;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x020000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x02000000;
+      break;
+
+    case 8:
+      w0[2] = 0x02;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0200;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x020000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x02000000;
+      break;
+
+    case 12:
+      w0[3] = 0x02;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0200;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x020000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x02000000;
+      break;
+  }
+}
+
+inline void append_0x02_2x4 (u32x w0[4], u32x w1[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x02;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0200;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x020000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x02000000;
+      break;
+
+    case 4:
+      w0[1] = 0x02;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0200;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x020000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x02000000;
+      break;
+
+    case 8:
+      w0[2] = 0x02;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0200;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x020000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x02000000;
+      break;
+
+    case 12:
+      w0[3] = 0x02;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0200;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x020000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x02000000;
+      break;
+
+    case 16:
+      w1[0] = 0x02;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0200;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x020000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x02000000;
+      break;
+
+    case 20:
+      w1[1] = 0x02;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0200;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x020000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x02000000;
+      break;
+
+    case 24:
+      w1[2] = 0x02;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0200;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x020000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x02000000;
+      break;
+
+    case 28:
+      w1[3] = 0x02;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0200;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x020000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x02000000;
+      break;
+  }
+}
+
+inline void append_0x02_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x02;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0200;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x020000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x02000000;
+      break;
+
+    case 4:
+      w0[1] = 0x02;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0200;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x020000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x02000000;
+      break;
+
+    case 8:
+      w0[2] = 0x02;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0200;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x020000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x02000000;
+      break;
+
+    case 12:
+      w0[3] = 0x02;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0200;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x020000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x02000000;
+      break;
+
+    case 16:
+      w1[0] = 0x02;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0200;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x020000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x02000000;
+      break;
+
+    case 20:
+      w1[1] = 0x02;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0200;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x020000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x02000000;
+      break;
+
+    case 24:
+      w1[2] = 0x02;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0200;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x020000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x02000000;
+      break;
+
+    case 28:
+      w1[3] = 0x02;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0200;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x020000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x02000000;
+      break;
+
+    case 32:
+      w2[0] = 0x02;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x0200;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x020000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x02000000;
+      break;
+
+    case 36:
+      w2[1] = 0x02;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x0200;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x020000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x02000000;
+      break;
+
+    case 40:
+      w2[2] = 0x02;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x0200;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x020000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x02000000;
+      break;
+
+    case 44:
+      w2[3] = 0x02;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x0200;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x020000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x02000000;
+      break;
+  }
+}
+
+inline void append_0x02_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x02;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0200;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x020000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x02000000;
+      break;
+
+    case 4:
+      w0[1] = 0x02;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0200;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x020000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x02000000;
+      break;
+
+    case 8:
+      w0[2] = 0x02;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0200;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x020000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x02000000;
+      break;
+
+    case 12:
+      w0[3] = 0x02;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0200;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x020000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x02000000;
+      break;
+
+    case 16:
+      w1[0] = 0x02;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0200;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x020000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x02000000;
+      break;
+
+    case 20:
+      w1[1] = 0x02;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0200;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x020000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x02000000;
+      break;
+
+    case 24:
+      w1[2] = 0x02;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0200;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x020000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x02000000;
+      break;
+
+    case 28:
+      w1[3] = 0x02;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0200;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x020000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x02000000;
+      break;
+
+    case 32:
+      w2[0] = 0x02;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x0200;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x020000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x02000000;
+      break;
+
+    case 36:
+      w2[1] = 0x02;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x0200;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x020000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x02000000;
+      break;
+
+    case 40:
+      w2[2] = 0x02;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x0200;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x020000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x02000000;
+      break;
+
+    case 44:
+      w2[3] = 0x02;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x0200;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x020000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x02000000;
+      break;
+
+    case 48:
+      w3[0] = 0x02;
+      break;
+
+    case 49:
+      w3[0] = w3[0] | 0x0200;
+      break;
+
+    case 50:
+      w3[0] = w3[0] | 0x020000;
+      break;
+
+    case 51:
+      w3[0] = w3[0] | 0x02000000;
+      break;
+
+    case 52:
+      w3[1] = 0x02;
+      break;
+
+    case 53:
+      w3[1] = w3[1] | 0x0200;
+      break;
+
+    case 54:
+      w3[1] = w3[1] | 0x020000;
+      break;
+
+    case 55:
+      w3[1] = w3[1] | 0x02000000;
+      break;
+
+    case 56:
+      w3[2] = 0x02;
+      break;
+
+    case 57:
+      w3[2] = w3[2] | 0x0200;
+      break;
+
+    case 58:
+      w3[2] = w3[2] | 0x020000;
+      break;
+
+    case 59:
+      w3[2] = w3[2] | 0x02000000;
+      break;
+
+    case 60:
+      w3[3] = 0x02;
+      break;
+
+    case 61:
+      w3[3] = w3[3] | 0x0200;
+      break;
+
+    case 62:
+      w3[3] = w3[3] | 0x020000;
+      break;
+
+    case 63:
+      w3[3] = w3[3] | 0x02000000;
+      break;
+  }
+}
+
+inline void append_0x02_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x02;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0200;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x020000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x02000000;
+      break;
+
+    case 4:
+      w0[1] = 0x02;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0200;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x020000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x02000000;
+      break;
+
+    case 8:
+      w0[2] = 0x02;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0200;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x020000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x02000000;
+      break;
+
+    case 12:
+      w0[3] = 0x02;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0200;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x020000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x02000000;
+      break;
+
+    case 16:
+      w1[0] = 0x02;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0200;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x020000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x02000000;
+      break;
+
+    case 20:
+      w1[1] = 0x02;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0200;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x020000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x02000000;
+      break;
+
+    case 24:
+      w1[2] = 0x02;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0200;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x020000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x02000000;
+      break;
+
+    case 28:
+      w1[3] = 0x02;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0200;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x020000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x02000000;
+      break;
+
+    case 32:
+      w2[0] = 0x02;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x0200;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x020000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x02000000;
+      break;
+
+    case 36:
+      w2[1] = 0x02;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x0200;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x020000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x02000000;
+      break;
+
+    case 40:
+      w2[2] = 0x02;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x0200;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x020000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x02000000;
+      break;
+
+    case 44:
+      w2[3] = 0x02;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x0200;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x020000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x02000000;
+      break;
+
+    case 48:
+      w3[0] = 0x02;
+      break;
+
+    case 49:
+      w3[0] = w3[0] | 0x0200;
+      break;
+
+    case 50:
+      w3[0] = w3[0] | 0x020000;
+      break;
+
+    case 51:
+      w3[0] = w3[0] | 0x02000000;
+      break;
+
+    case 52:
+      w3[1] = 0x02;
+      break;
+
+    case 53:
+      w3[1] = w3[1] | 0x0200;
+      break;
+
+    case 54:
+      w3[1] = w3[1] | 0x020000;
+      break;
+
+    case 55:
+      w3[1] = w3[1] | 0x02000000;
+      break;
+
+    case 56:
+      w3[2] = 0x02;
+      break;
+
+    case 57:
+      w3[2] = w3[2] | 0x0200;
+      break;
+
+    case 58:
+      w3[2] = w3[2] | 0x020000;
+      break;
+
+    case 59:
+      w3[2] = w3[2] | 0x02000000;
+      break;
+
+    case 60:
+      w3[3] = 0x02;
+      break;
+
+    case 61:
+      w3[3] = w3[3] | 0x0200;
+      break;
+
+    case 62:
+      w3[3] = w3[3] | 0x020000;
+      break;
+
+    case 63:
+      w3[3] = w3[3] | 0x02000000;
+      break;
+
+    case 64:
+      w4[0] = 0x02;
+      break;
+
+    case 65:
+      w4[0] = w4[0] | 0x0200;
+      break;
+
+    case 66:
+      w4[0] = w4[0] | 0x020000;
+      break;
+
+    case 67:
+      w4[0] = w4[0] | 0x02000000;
+      break;
+
+    case 68:
+      w4[1] = 0x02;
+      break;
+
+    case 69:
+      w4[1] = w4[1] | 0x0200;
+      break;
+
+    case 70:
+      w4[1] = w4[1] | 0x020000;
+      break;
+
+    case 71:
+      w4[1] = w4[1] | 0x02000000;
+      break;
+
+    case 72:
+      w4[2] = 0x02;
+      break;
+
+    case 73:
+      w4[2] = w4[2] | 0x0200;
+      break;
+
+    case 74:
+      w4[2] = w4[2] | 0x020000;
+      break;
+
+    case 75:
+      w4[2] = w4[2] | 0x02000000;
+      break;
+
+    case 76:
+      w4[3] = 0x02;
+      break;
+
+    case 77:
+      w4[3] = w4[3] | 0x0200;
+      break;
+
+    case 78:
+      w4[3] = w4[3] | 0x020000;
+      break;
+
+    case 79:
+      w4[3] = w4[3] | 0x02000000;
+      break;
+
+    case 80:
+      w5[0] = 0x02;
+      break;
+
+    case 81:
+      w5[0] = w5[0] | 0x0200;
+      break;
+
+    case 82:
+      w5[0] = w5[0] | 0x020000;
+      break;
+
+    case 83:
+      w5[0] = w5[0] | 0x02000000;
+      break;
+
+    case 84:
+      w5[1] = 0x02;
+      break;
+
+    case 85:
+      w5[1] = w5[1] | 0x0200;
+      break;
+
+    case 86:
+      w5[1] = w5[1] | 0x020000;
+      break;
+
+    case 87:
+      w5[1] = w5[1] | 0x02000000;
+      break;
+
+    case 88:
+      w5[2] = 0x02;
+      break;
+
+    case 89:
+      w5[2] = w5[2] | 0x0200;
+      break;
+
+    case 90:
+      w5[2] = w5[2] | 0x020000;
+      break;
+
+    case 91:
+      w5[2] = w5[2] | 0x02000000;
+      break;
+
+    case 92:
+      w5[3] = 0x02;
+      break;
+
+    case 93:
+      w5[3] = w5[3] | 0x0200;
+      break;
+
+    case 94:
+      w5[3] = w5[3] | 0x020000;
+      break;
+
+    case 95:
+      w5[3] = w5[3] | 0x02000000;
+      break;
+
+    case 96:
+      w6[0] = 0x02;
+      break;
+
+    case 97:
+      w6[0] = w6[0] | 0x0200;
+      break;
+
+    case 98:
+      w6[0] = w6[0] | 0x020000;
+      break;
+
+    case 99:
+      w6[0] = w6[0] | 0x02000000;
+      break;
+
+    case 100:
+      w6[1] = 0x02;
+      break;
+
+    case 101:
+      w6[1] = w6[1] | 0x0200;
+      break;
+
+    case 102:
+      w6[1] = w6[1] | 0x020000;
+      break;
+
+    case 103:
+      w6[1] = w6[1] | 0x02000000;
+      break;
+
+    case 104:
+      w6[2] = 0x02;
+      break;
+
+    case 105:
+      w6[2] = w6[2] | 0x0200;
+      break;
+
+    case 106:
+      w6[2] = w6[2] | 0x020000;
+      break;
+
+    case 107:
+      w6[2] = w6[2] | 0x02000000;
+      break;
+
+    case 108:
+      w6[3] = 0x02;
+      break;
+
+    case 109:
+      w6[3] = w6[3] | 0x0200;
+      break;
+
+    case 110:
+      w6[3] = w6[3] | 0x020000;
+      break;
+
+    case 111:
+      w6[3] = w6[3] | 0x02000000;
+      break;
+
+    case 112:
+      w7[0] = 0x02;
+      break;
+
+    case 113:
+      w7[0] = w7[0] | 0x0200;
+      break;
+
+    case 114:
+      w7[0] = w7[0] | 0x020000;
+      break;
+
+    case 115:
+      w7[0] = w7[0] | 0x02000000;
+      break;
+
+    case 116:
+      w7[1] = 0x02;
+      break;
+
+    case 117:
+      w7[1] = w7[1] | 0x0200;
+      break;
+
+    case 118:
+      w7[1] = w7[1] | 0x020000;
+      break;
+
+    case 119:
+      w7[1] = w7[1] | 0x02000000;
+      break;
+
+    case 120:
+      w7[2] = 0x02;
+      break;
+
+    case 121:
+      w7[2] = w7[2] | 0x0200;
+      break;
+
+    case 122:
+      w7[2] = w7[2] | 0x020000;
+      break;
+
+    case 123:
+      w7[2] = w7[2] | 0x02000000;
+      break;
+
+    case 124:
+      w7[3] = 0x02;
+      break;
+
+    case 125:
+      w7[3] = w7[3] | 0x0200;
+      break;
+
+    case 126:
+      w7[3] = w7[3] | 0x020000;
+      break;
+
+    case 127:
+      w7[3] = w7[3] | 0x02000000;
+      break;
+  }
+}
+
+inline void append_0x80_1x4 (u32x w0[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0]  = 0x80;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x8000;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x800000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x80000000;
+      break;
+
+    case 4:
+      w0[1] = 0x80;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x8000;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x800000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x80000000;
+      break;
+
+    case 8:
+      w0[2] = 0x80;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x8000;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x800000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x80000000;
+      break;
+
+    case 12:
+      w0[3] = 0x80;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x8000;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x800000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x80000000;
+      break;
+  }
+}
+
+inline void append_0x80_2x4 (u32x w0[4], u32x w1[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x80;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x8000;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x800000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x80000000;
+      break;
+
+    case 4:
+      w0[1] = 0x80;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x8000;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x800000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x80000000;
+      break;
+
+    case 8:
+      w0[2] = 0x80;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x8000;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x800000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x80000000;
+      break;
+
+    case 12:
+      w0[3] = 0x80;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x8000;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x800000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x80000000;
+      break;
+
+    case 16:
+      w1[0] = 0x80;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x8000;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x800000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x80000000;
+      break;
+
+    case 20:
+      w1[1] = 0x80;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x8000;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x800000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x80000000;
+      break;
+
+    case 24:
+      w1[2] = 0x80;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x8000;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x800000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x80000000;
+      break;
+
+    case 28:
+      w1[3] = 0x80;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x8000;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x800000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x80000000;
+      break;
+  }
+}
+
+inline void append_0x80_3x4 (u32x w0[4], u32x w1[4], u32x w2[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x80;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x8000;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x800000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x80000000;
+      break;
+
+    case 4:
+      w0[1] = 0x80;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x8000;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x800000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x80000000;
+      break;
+
+    case 8:
+      w0[2] = 0x80;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x8000;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x800000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x80000000;
+      break;
+
+    case 12:
+      w0[3] = 0x80;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x8000;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x800000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x80000000;
+      break;
+
+    case 16:
+      w1[0] = 0x80;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x8000;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x800000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x80000000;
+      break;
+
+    case 20:
+      w1[1] = 0x80;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x8000;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x800000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x80000000;
+      break;
+
+    case 24:
+      w1[2] = 0x80;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x8000;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x800000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x80000000;
+      break;
+
+    case 28:
+      w1[3] = 0x80;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x8000;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x800000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x80000000;
+      break;
+
+    case 32:
+      w2[0] = 0x80;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x8000;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x800000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x80000000;
+      break;
+
+    case 36:
+      w2[1] = 0x80;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x8000;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x800000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x80000000;
+      break;
+
+    case 40:
+      w2[2] = 0x80;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x8000;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x800000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x80000000;
+      break;
+
+    case 44:
+      w2[3] = 0x80;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x8000;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x800000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x80000000;
+      break;
+  }
+}
+
+inline void append_0x80_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x80;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x8000;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x800000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x80000000;
+      break;
+
+    case 4:
+      w0[1] = 0x80;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x8000;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x800000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x80000000;
+      break;
+
+    case 8:
+      w0[2] = 0x80;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x8000;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x800000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x80000000;
+      break;
+
+    case 12:
+      w0[3] = 0x80;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x8000;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x800000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x80000000;
+      break;
+
+    case 16:
+      w1[0] = 0x80;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x8000;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x800000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x80000000;
+      break;
+
+    case 20:
+      w1[1] = 0x80;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x8000;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x800000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x80000000;
+      break;
+
+    case 24:
+      w1[2] = 0x80;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x8000;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x800000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x80000000;
+      break;
+
+    case 28:
+      w1[3] = 0x80;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x8000;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x800000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x80000000;
+      break;
+
+    case 32:
+      w2[0] = 0x80;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x8000;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x800000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x80000000;
+      break;
+
+    case 36:
+      w2[1] = 0x80;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x8000;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x800000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x80000000;
+      break;
+
+    case 40:
+      w2[2] = 0x80;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x8000;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x800000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x80000000;
+      break;
+
+    case 44:
+      w2[3] = 0x80;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x8000;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x800000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x80000000;
+      break;
+
+    case 48:
+      w3[0] = 0x80;
+      break;
+
+    case 49:
+      w3[0] = w3[0] | 0x8000;
+      break;
+
+    case 50:
+      w3[0] = w3[0] | 0x800000;
+      break;
+
+    case 51:
+      w3[0] = w3[0] | 0x80000000;
+      break;
+
+    case 52:
+      w3[1] = 0x80;
+      break;
+
+    case 53:
+      w3[1] = w3[1] | 0x8000;
+      break;
+
+    case 54:
+      w3[1] = w3[1] | 0x800000;
+      break;
+
+    case 55:
+      w3[1] = w3[1] | 0x80000000;
+      break;
+
+    case 56:
+      w3[2] = 0x80;
+      break;
+
+    case 57:
+      w3[2] = w3[2] | 0x8000;
+      break;
+
+    case 58:
+      w3[2] = w3[2] | 0x800000;
+      break;
+
+    case 59:
+      w3[2] = w3[2] | 0x80000000;
+      break;
+
+    case 60:
+      w3[3] = 0x80;
+      break;
+
+    case 61:
+      w3[3] = w3[3] | 0x8000;
+      break;
+
+    case 62:
+      w3[3] = w3[3] | 0x800000;
+      break;
+
+    case 63:
+      w3[3] = w3[3] | 0x80000000;
+      break;
+  }
+}
+
+inline void append_0x80_8x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x w4[4], u32x w5[4], u32x w6[4], u32x w7[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x80;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x8000;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x800000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x80000000;
+      break;
+
+    case 4:
+      w0[1] = 0x80;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x8000;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x800000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x80000000;
+      break;
+
+    case 8:
+      w0[2] = 0x80;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x8000;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x800000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x80000000;
+      break;
+
+    case 12:
+      w0[3] = 0x80;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x8000;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x800000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x80000000;
+      break;
+
+    case 16:
+      w1[0] = 0x80;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x8000;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x800000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x80000000;
+      break;
+
+    case 20:
+      w1[1] = 0x80;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x8000;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x800000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x80000000;
+      break;
+
+    case 24:
+      w1[2] = 0x80;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x8000;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x800000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x80000000;
+      break;
+
+    case 28:
+      w1[3] = 0x80;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x8000;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x800000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x80000000;
+      break;
+
+    case 32:
+      w2[0] = 0x80;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x8000;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x800000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x80000000;
+      break;
+
+    case 36:
+      w2[1] = 0x80;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x8000;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x800000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x80000000;
+      break;
+
+    case 40:
+      w2[2] = 0x80;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x8000;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x800000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x80000000;
+      break;
+
+    case 44:
+      w2[3] = 0x80;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x8000;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x800000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x80000000;
+      break;
+
+    case 48:
+      w3[0] = 0x80;
+      break;
+
+    case 49:
+      w3[0] = w3[0] | 0x8000;
+      break;
+
+    case 50:
+      w3[0] = w3[0] | 0x800000;
+      break;
+
+    case 51:
+      w3[0] = w3[0] | 0x80000000;
+      break;
+
+    case 52:
+      w3[1] = 0x80;
+      break;
+
+    case 53:
+      w3[1] = w3[1] | 0x8000;
+      break;
+
+    case 54:
+      w3[1] = w3[1] | 0x800000;
+      break;
+
+    case 55:
+      w3[1] = w3[1] | 0x80000000;
+      break;
+
+    case 56:
+      w3[2] = 0x80;
+      break;
+
+    case 57:
+      w3[2] = w3[2] | 0x8000;
+      break;
+
+    case 58:
+      w3[2] = w3[2] | 0x800000;
+      break;
+
+    case 59:
+      w3[2] = w3[2] | 0x80000000;
+      break;
+
+    case 60:
+      w3[3] = 0x80;
+      break;
+
+    case 61:
+      w3[3] = w3[3] | 0x8000;
+      break;
+
+    case 62:
+      w3[3] = w3[3] | 0x800000;
+      break;
+
+    case 63:
+      w3[3] = w3[3] | 0x80000000;
+      break;
+
+    case 64:
+      w4[0] = 0x80;
+      break;
+
+    case 65:
+      w4[0] = w4[0] | 0x8000;
+      break;
+
+    case 66:
+      w4[0] = w4[0] | 0x800000;
+      break;
+
+    case 67:
+      w4[0] = w4[0] | 0x80000000;
+      break;
+
+    case 68:
+      w4[1] = 0x80;
+      break;
+
+    case 69:
+      w4[1] = w4[1] | 0x8000;
+      break;
+
+    case 70:
+      w4[1] = w4[1] | 0x800000;
+      break;
+
+    case 71:
+      w4[1] = w4[1] | 0x80000000;
+      break;
+
+    case 72:
+      w4[2] = 0x80;
+      break;
+
+    case 73:
+      w4[2] = w4[2] | 0x8000;
+      break;
+
+    case 74:
+      w4[2] = w4[2] | 0x800000;
+      break;
+
+    case 75:
+      w4[2] = w4[2] | 0x80000000;
+      break;
+
+    case 76:
+      w4[3] = 0x80;
+      break;
+
+    case 77:
+      w4[3] = w4[3] | 0x8000;
+      break;
+
+    case 78:
+      w4[3] = w4[3] | 0x800000;
+      break;
+
+    case 79:
+      w4[3] = w4[3] | 0x80000000;
+      break;
+
+    case 80:
+      w5[0] = 0x80;
+      break;
+
+    case 81:
+      w5[0] = w5[0] | 0x8000;
+      break;
+
+    case 82:
+      w5[0] = w5[0] | 0x800000;
+      break;
+
+    case 83:
+      w5[0] = w5[0] | 0x80000000;
+      break;
+
+    case 84:
+      w5[1] = 0x80;
+      break;
+
+    case 85:
+      w5[1] = w5[1] | 0x8000;
+      break;
+
+    case 86:
+      w5[1] = w5[1] | 0x800000;
+      break;
+
+    case 87:
+      w5[1] = w5[1] | 0x80000000;
+      break;
+
+    case 88:
+      w5[2] = 0x80;
+      break;
+
+    case 89:
+      w5[2] = w5[2] | 0x8000;
+      break;
+
+    case 90:
+      w5[2] = w5[2] | 0x800000;
+      break;
+
+    case 91:
+      w5[2] = w5[2] | 0x80000000;
+      break;
+
+    case 92:
+      w5[3] = 0x80;
+      break;
+
+    case 93:
+      w5[3] = w5[3] | 0x8000;
+      break;
+
+    case 94:
+      w5[3] = w5[3] | 0x800000;
+      break;
+
+    case 95:
+      w5[3] = w5[3] | 0x80000000;
+      break;
+
+    case 96:
+      w6[0] = 0x80;
+      break;
+
+    case 97:
+      w6[0] = w6[0] | 0x8000;
+      break;
+
+    case 98:
+      w6[0] = w6[0] | 0x800000;
+      break;
+
+    case 99:
+      w6[0] = w6[0] | 0x80000000;
+      break;
+
+    case 100:
+      w6[1] = 0x80;
+      break;
+
+    case 101:
+      w6[1] = w6[1] | 0x8000;
+      break;
+
+    case 102:
+      w6[1] = w6[1] | 0x800000;
+      break;
+
+    case 103:
+      w6[1] = w6[1] | 0x80000000;
+      break;
+
+    case 104:
+      w6[2] = 0x80;
+      break;
+
+    case 105:
+      w6[2] = w6[2] | 0x8000;
+      break;
+
+    case 106:
+      w6[2] = w6[2] | 0x800000;
+      break;
+
+    case 107:
+      w6[2] = w6[2] | 0x80000000;
+      break;
+
+    case 108:
+      w6[3] = 0x80;
+      break;
+
+    case 109:
+      w6[3] = w6[3] | 0x8000;
+      break;
+
+    case 110:
+      w6[3] = w6[3] | 0x800000;
+      break;
+
+    case 111:
+      w6[3] = w6[3] | 0x80000000;
+      break;
+
+    case 112:
+      w7[0] = 0x80;
+      break;
+
+    case 113:
+      w7[0] = w7[0] | 0x8000;
+      break;
+
+    case 114:
+      w7[0] = w7[0] | 0x800000;
+      break;
+
+    case 115:
+      w7[0] = w7[0] | 0x80000000;
+      break;
+
+    case 116:
+      w7[1] = 0x80;
+      break;
+
+    case 117:
+      w7[1] = w7[1] | 0x8000;
+      break;
+
+    case 118:
+      w7[1] = w7[1] | 0x800000;
+      break;
+
+    case 119:
+      w7[1] = w7[1] | 0x80000000;
+      break;
+
+    case 120:
+      w7[2] = 0x80;
+      break;
+
+    case 121:
+      w7[2] = w7[2] | 0x8000;
+      break;
+
+    case 122:
+      w7[2] = w7[2] | 0x800000;
+      break;
+
+    case 123:
+      w7[2] = w7[2] | 0x80000000;
+      break;
+
+    case 124:
+      w7[3] = 0x80;
+      break;
+
+    case 125:
+      w7[3] = w7[3] | 0x8000;
+      break;
+
+    case 126:
+      w7[3] = w7[3] | 0x800000;
+      break;
+
+    case 127:
+      w7[3] = w7[3] | 0x80000000;
+      break;
+  }
+}
+
+inline void append_0x80_1x16 (u32x w[16], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w[ 0] = 0x80;
+      break;
+
+    case 1:
+      w[ 0] = w[ 0] | 0x8000;
+      break;
+
+    case 2:
+      w[ 0] = w[ 0] | 0x800000;
+      break;
+
+    case 3:
+      w[ 0] = w[ 0] | 0x80000000;
+      break;
+
+    case 4:
+      w[ 1] = 0x80;
+      break;
+
+    case 5:
+      w[ 1] = w[ 1] | 0x8000;
+      break;
+
+    case 6:
+      w[ 1] = w[ 1] | 0x800000;
+      break;
+
+    case 7:
+      w[ 1] = w[ 1] | 0x80000000;
+      break;
+
+    case 8:
+      w[ 2] = 0x80;
+      break;
+
+    case 9:
+      w[ 2] = w[ 2] | 0x8000;
+      break;
+
+    case 10:
+      w[ 2] = w[ 2] | 0x800000;
+      break;
+
+    case 11:
+      w[ 2] = w[ 2] | 0x80000000;
+      break;
+
+    case 12:
+      w[ 3] = 0x80;
+      break;
+
+    case 13:
+      w[ 3] = w[ 3] | 0x8000;
+      break;
+
+    case 14:
+      w[ 3] = w[ 3] | 0x800000;
+      break;
+
+    case 15:
+      w[ 3] = w[ 3] | 0x80000000;
+      break;
+
+    case 16:
+      w[ 4] = 0x80;
+      break;
+
+    case 17:
+      w[ 4] = w[ 4] | 0x8000;
+      break;
+
+    case 18:
+      w[ 4] = w[ 4] | 0x800000;
+      break;
+
+    case 19:
+      w[ 4] = w[ 4] | 0x80000000;
+      break;
+
+    case 20:
+      w[ 5] = 0x80;
+      break;
+
+    case 21:
+      w[ 5] = w[ 5] | 0x8000;
+      break;
+
+    case 22:
+      w[ 5] = w[ 5] | 0x800000;
+      break;
+
+    case 23:
+      w[ 5] = w[ 5] | 0x80000000;
+      break;
+
+    case 24:
+      w[ 6] = 0x80;
+      break;
+
+    case 25:
+      w[ 6] = w[ 6] | 0x8000;
+      break;
+
+    case 26:
+      w[ 6] = w[ 6] | 0x800000;
+      break;
+
+    case 27:
+      w[ 6] = w[ 6] | 0x80000000;
+      break;
+
+    case 28:
+      w[ 7] = 0x80;
+      break;
+
+    case 29:
+      w[ 7] = w[ 7] | 0x8000;
+      break;
+
+    case 30:
+      w[ 7] = w[ 7] | 0x800000;
+      break;
+
+    case 31:
+      w[ 7] = w[ 7] | 0x80000000;
+      break;
+
+    case 32:
+      w[ 8] = 0x80;
+      break;
+
+    case 33:
+      w[ 8] = w[ 8] | 0x8000;
+      break;
+
+    case 34:
+      w[ 8] = w[ 8] | 0x800000;
+      break;
+
+    case 35:
+      w[ 8] = w[ 8] | 0x80000000;
+      break;
+
+    case 36:
+      w[ 9] = 0x80;
+      break;
+
+    case 37:
+      w[ 9] = w[ 9] | 0x8000;
+      break;
+
+    case 38:
+      w[ 9] = w[ 9] | 0x800000;
+      break;
+
+    case 39:
+      w[ 9] = w[ 9] | 0x80000000;
+      break;
+
+    case 40:
+      w[10] = 0x80;
+      break;
+
+    case 41:
+      w[10] = w[10] | 0x8000;
+      break;
+
+    case 42:
+      w[10] = w[10] | 0x800000;
+      break;
+
+    case 43:
+      w[10] = w[10] | 0x80000000;
+      break;
+
+    case 44:
+      w[11] = 0x80;
+      break;
+
+    case 45:
+      w[11] = w[11] | 0x8000;
+      break;
+
+    case 46:
+      w[11] = w[11] | 0x800000;
+      break;
+
+    case 47:
+      w[11] = w[11] | 0x80000000;
+      break;
+
+    case 48:
+      w[12] = 0x80;
+      break;
+
+    case 49:
+      w[12] = w[12] | 0x8000;
+      break;
+
+    case 50:
+      w[12] = w[12] | 0x800000;
+      break;
+
+    case 51:
+      w[12] = w[12] | 0x80000000;
+      break;
+
+    case 52:
+      w[13] = 0x80;
+      break;
+
+    case 53:
+      w[13] = w[13] | 0x8000;
+      break;
+
+    case 54:
+      w[13] = w[13] | 0x800000;
+      break;
+
+    case 55:
+      w[13] = w[13] | 0x80000000;
+      break;
+
+    case 56:
+      w[14] = 0x80;
+      break;
+
+    case 57:
+      w[14] = w[14] | 0x8000;
+      break;
+
+    case 58:
+      w[14] = w[14] | 0x800000;
+      break;
+
+    case 59:
+      w[14] = w[14] | 0x80000000;
+      break;
+
+    case 60:
+      w[15] = 0x80;
+      break;
+
+    case 61:
+      w[15] = w[15] | 0x8000;
+      break;
+
+    case 62:
+      w[15] = w[15] | 0x800000;
+      break;
+
+    case 63:
+      w[15] = w[15] | 0x80000000;
+      break;
+  }
+}
+
+inline void switch_buffer_by_offset_le (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
+{
+  #if defined IS_AMD || defined IS_GENERIC
+  const int offset_mod_4 = offset & 3;
+
+  const int offset_minus_4 = 4 - offset;
+
+  switch (offset / 4)
+  {
+    case 0:
+      w3[2] = amd_bytealign (    0, w3[1], offset_minus_4);
+      w3[1] = amd_bytealign (w3[1], w3[0], offset_minus_4);
+      w3[0] = amd_bytealign (w3[0], w2[3], offset_minus_4);
+      w2[3] = amd_bytealign (w2[3], w2[2], offset_minus_4);
+      w2[2] = amd_bytealign (w2[2], w2[1], offset_minus_4);
+      w2[1] = amd_bytealign (w2[1], w2[0], offset_minus_4);
+      w2[0] = amd_bytealign (w2[0], w1[3], offset_minus_4);
+      w1[3] = amd_bytealign (w1[3], w1[2], offset_minus_4);
+      w1[2] = amd_bytealign (w1[2], w1[1], offset_minus_4);
+      w1[1] = amd_bytealign (w1[1], w1[0], offset_minus_4);
+      w1[0] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w0[3] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w0[2] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w0[1] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w0[0] = amd_bytealign (w0[0],     0, offset_minus_4);
+
+      if (offset_mod_4 == 0)
+      {
+        w0[0] = w0[1];
+        w0[1] = w0[2];
+        w0[2] = w0[3];
+        w0[3] = w1[0];
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 1:
+      w3[2] = amd_bytealign (    0, w3[0], offset_minus_4);
+      w3[1] = amd_bytealign (w3[0], w2[3], offset_minus_4);
+      w3[0] = amd_bytealign (w2[3], w2[2], offset_minus_4);
+      w2[3] = amd_bytealign (w2[2], w2[1], offset_minus_4);
+      w2[2] = amd_bytealign (w2[1], w2[0], offset_minus_4);
+      w2[1] = amd_bytealign (w2[0], w1[3], offset_minus_4);
+      w2[0] = amd_bytealign (w1[3], w1[2], offset_minus_4);
+      w1[3] = amd_bytealign (w1[2], w1[1], offset_minus_4);
+      w1[2] = amd_bytealign (w1[1], w1[0], offset_minus_4);
+      w1[1] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w1[0] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w0[3] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w0[2] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w0[1] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w0[1] = w0[2];
+        w0[2] = w0[3];
+        w0[3] = w1[0];
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 2:
+      w3[2] = amd_bytealign (    0, w2[3], offset_minus_4);
+      w3[1] = amd_bytealign (w2[3], w2[2], offset_minus_4);
+      w3[0] = amd_bytealign (w2[2], w2[1], offset_minus_4);
+      w2[3] = amd_bytealign (w2[1], w2[0], offset_minus_4);
+      w2[2] = amd_bytealign (w2[0], w1[3], offset_minus_4);
+      w2[1] = amd_bytealign (w1[3], w1[2], offset_minus_4);
+      w2[0] = amd_bytealign (w1[2], w1[1], offset_minus_4);
+      w1[3] = amd_bytealign (w1[1], w1[0], offset_minus_4);
+      w1[2] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w1[1] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w1[0] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w0[3] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w0[2] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w0[2] = w0[3];
+        w0[3] = w1[0];
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 3:
+      w3[2] = amd_bytealign (    0, w2[2], offset_minus_4);
+      w3[1] = amd_bytealign (w2[2], w2[1], offset_minus_4);
+      w3[0] = amd_bytealign (w2[1], w2[0], offset_minus_4);
+      w2[3] = amd_bytealign (w2[0], w1[3], offset_minus_4);
+      w2[2] = amd_bytealign (w1[3], w1[2], offset_minus_4);
+      w2[1] = amd_bytealign (w1[2], w1[1], offset_minus_4);
+      w2[0] = amd_bytealign (w1[1], w1[0], offset_minus_4);
+      w1[3] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w1[2] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w1[1] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w1[0] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w0[3] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w0[3] = w1[0];
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 4:
+      w3[2] = amd_bytealign (    0, w2[1], offset_minus_4);
+      w3[1] = amd_bytealign (w2[1], w2[0], offset_minus_4);
+      w3[0] = amd_bytealign (w2[0], w1[3], offset_minus_4);
+      w2[3] = amd_bytealign (w1[3], w1[2], offset_minus_4);
+      w2[2] = amd_bytealign (w1[2], w1[1], offset_minus_4);
+      w2[1] = amd_bytealign (w1[1], w1[0], offset_minus_4);
+      w2[0] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w1[3] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w1[2] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w1[1] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w1[0] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 5:
+      w3[2] = amd_bytealign (    0, w2[0], offset_minus_4);
+      w3[1] = amd_bytealign (w2[0], w1[3], offset_minus_4);
+      w3[0] = amd_bytealign (w1[3], w1[2], offset_minus_4);
+      w2[3] = amd_bytealign (w1[2], w1[1], offset_minus_4);
+      w2[2] = amd_bytealign (w1[1], w1[0], offset_minus_4);
+      w2[1] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w2[0] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w1[3] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w1[2] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w1[1] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 6:
+      w3[2] = amd_bytealign (    0, w1[3], offset_minus_4);
+      w3[1] = amd_bytealign (w1[3], w1[2], offset_minus_4);
+      w3[0] = amd_bytealign (w1[2], w1[1], offset_minus_4);
+      w2[3] = amd_bytealign (w1[1], w1[0], offset_minus_4);
+      w2[2] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w2[1] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w2[0] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w1[3] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w1[2] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 7:
+      w3[2] = amd_bytealign (    0, w1[2], offset_minus_4);
+      w3[1] = amd_bytealign (w1[2], w1[1], offset_minus_4);
+      w3[0] = amd_bytealign (w1[1], w1[0], offset_minus_4);
+      w2[3] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w2[2] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w2[1] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w2[0] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w1[3] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 8:
+      w3[2] = amd_bytealign (    0, w1[1], offset_minus_4);
+      w3[1] = amd_bytealign (w1[1], w1[0], offset_minus_4);
+      w3[0] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w2[3] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w2[2] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w2[1] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w2[0] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 9:
+      w3[2] = amd_bytealign (    0, w1[0], offset_minus_4);
+      w3[1] = amd_bytealign (w1[0], w0[3], offset_minus_4);
+      w3[0] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w2[3] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w2[2] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w2[1] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 10:
+      w3[2] = amd_bytealign (    0, w0[3], offset_minus_4);
+      w3[1] = amd_bytealign (w0[3], w0[2], offset_minus_4);
+      w3[0] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w2[3] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w2[2] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 11:
+      w3[2] = amd_bytealign (    0, w0[2], offset_minus_4);
+      w3[1] = amd_bytealign (w0[2], w0[1], offset_minus_4);
+      w3[0] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w2[3] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 12:
+      w3[2] = amd_bytealign (    0, w0[1], offset_minus_4);
+      w3[1] = amd_bytealign (w0[1], w0[0], offset_minus_4);
+      w3[0] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 13:
+      w3[2] = amd_bytealign (    0, w0[0], offset_minus_4);
+      w3[1] = amd_bytealign (w0[0],     0, offset_minus_4);
+      w3[0] = 0;
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+  }
+  #endif
+
+  #ifdef IS_NV
+  const int offset_minus_4 = 4 - (offset % 4);
+
+  const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff;
+
+  switch (offset / 4)
+  {
+    case 0:
+      w3[1] = __byte_perm (w3[0], w3[1], selector);
+      w3[0] = __byte_perm (w2[3], w3[0], selector);
+      w2[3] = __byte_perm (w2[2], w2[3], selector);
+      w2[2] = __byte_perm (w2[1], w2[2], selector);
+      w2[1] = __byte_perm (w2[0], w2[1], selector);
+      w2[0] = __byte_perm (w1[3], w2[0], selector);
+      w1[3] = __byte_perm (w1[2], w1[3], selector);
+      w1[2] = __byte_perm (w1[1], w1[2], selector);
+      w1[1] = __byte_perm (w1[0], w1[1], selector);
+      w1[0] = __byte_perm (w0[3], w1[0], selector);
+      w0[3] = __byte_perm (w0[2], w0[3], selector);
+      w0[2] = __byte_perm (w0[1], w0[2], selector);
+      w0[1] = __byte_perm (w0[0], w0[1], selector);
+      w0[0] = __byte_perm (    0, w0[0], selector);
+
+      break;
+
+    case 1:
+      w3[1] = __byte_perm (w2[3], w3[0], selector);
+      w3[0] = __byte_perm (w2[2], w2[3], selector);
+      w2[3] = __byte_perm (w2[1], w2[2], selector);
+      w2[2] = __byte_perm (w2[0], w2[1], selector);
+      w2[1] = __byte_perm (w1[3], w2[0], selector);
+      w2[0] = __byte_perm (w1[2], w1[3], selector);
+      w1[3] = __byte_perm (w1[1], w1[2], selector);
+      w1[2] = __byte_perm (w1[0], w1[1], selector);
+      w1[1] = __byte_perm (w0[3], w1[0], selector);
+      w1[0] = __byte_perm (w0[2], w0[3], selector);
+      w0[3] = __byte_perm (w0[1], w0[2], selector);
+      w0[2] = __byte_perm (w0[0], w0[1], selector);
+      w0[1] = __byte_perm (    0, w0[0], selector);
+      w0[0] = 0;
+
+      break;
+
+    case 2:
+      w3[1] = __byte_perm (w2[2], w2[3], selector);
+      w3[0] = __byte_perm (w2[1], w2[2], selector);
+      w2[3] = __byte_perm (w2[0], w2[1], selector);
+      w2[2] = __byte_perm (w1[3], w2[0], selector);
+      w2[1] = __byte_perm (w1[2], w1[3], selector);
+      w2[0] = __byte_perm (w1[1], w1[2], selector);
+      w1[3] = __byte_perm (w1[0], w1[1], selector);
+      w1[2] = __byte_perm (w0[3], w1[0], selector);
+      w1[1] = __byte_perm (w0[2], w0[3], selector);
+      w1[0] = __byte_perm (w0[1], w0[2], selector);
+      w0[3] = __byte_perm (w0[0], w0[1], selector);
+      w0[2] = __byte_perm (    0, w0[0], selector);
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 3:
+      w3[1] = __byte_perm (w2[1], w2[2], selector);
+      w3[0] = __byte_perm (w2[0], w2[1], selector);
+      w2[3] = __byte_perm (w1[3], w2[0], selector);
+      w2[2] = __byte_perm (w1[2], w1[3], selector);
+      w2[1] = __byte_perm (w1[1], w1[2], selector);
+      w2[0] = __byte_perm (w1[0], w1[1], selector);
+      w1[3] = __byte_perm (w0[3], w1[0], selector);
+      w1[2] = __byte_perm (w0[2], w0[3], selector);
+      w1[1] = __byte_perm (w0[1], w0[2], selector);
+      w1[0] = __byte_perm (w0[0], w0[1], selector);
+      w0[3] = __byte_perm (    0, w0[0], selector);
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 4:
+      w3[1] = __byte_perm (w2[0], w2[1], selector);
+      w3[0] = __byte_perm (w1[3], w2[0], selector);
+      w2[3] = __byte_perm (w1[2], w1[3], selector);
+      w2[2] = __byte_perm (w1[1], w1[2], selector);
+      w2[1] = __byte_perm (w1[0], w1[1], selector);
+      w2[0] = __byte_perm (w0[3], w1[0], selector);
+      w1[3] = __byte_perm (w0[2], w0[3], selector);
+      w1[2] = __byte_perm (w0[1], w0[2], selector);
+      w1[1] = __byte_perm (w0[0], w0[1], selector);
+      w1[0] = __byte_perm (    0, w0[0], selector);
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 5:
+      w3[1] = __byte_perm (w1[3], w2[0], selector);
+      w3[0] = __byte_perm (w1[2], w1[3], selector);
+      w2[3] = __byte_perm (w1[1], w1[2], selector);
+      w2[2] = __byte_perm (w1[0], w1[1], selector);
+      w2[1] = __byte_perm (w0[3], w1[0], selector);
+      w2[0] = __byte_perm (w0[2], w0[3], selector);
+      w1[3] = __byte_perm (w0[1], w0[2], selector);
+      w1[2] = __byte_perm (w0[0], w0[1], selector);
+      w1[1] = __byte_perm (    0, w0[0], selector);
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 6:
+      w3[1] = __byte_perm (w1[2], w1[3], selector);
+      w3[0] = __byte_perm (w1[1], w1[2], selector);
+      w2[3] = __byte_perm (w1[0], w1[1], selector);
+      w2[2] = __byte_perm (w0[3], w1[0], selector);
+      w2[1] = __byte_perm (w0[2], w0[3], selector);
+      w2[0] = __byte_perm (w0[1], w0[2], selector);
+      w1[3] = __byte_perm (w0[0], w0[1], selector);
+      w1[2] = __byte_perm (    0, w0[0], selector);
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 7:
+      w3[1] = __byte_perm (w1[1], w1[2], selector);
+      w3[0] = __byte_perm (w1[0], w1[1], selector);
+      w2[3] = __byte_perm (w0[3], w1[0], selector);
+      w2[2] = __byte_perm (w0[2], w0[3], selector);
+      w2[1] = __byte_perm (w0[1], w0[2], selector);
+      w2[0] = __byte_perm (w0[0], w0[1], selector);
+      w1[3] = __byte_perm (    0, w0[0], selector);
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 8:
+      w3[1] = __byte_perm (w1[0], w1[1], selector);
+      w3[0] = __byte_perm (w0[3], w1[0], selector);
+      w2[3] = __byte_perm (w0[2], w0[3], selector);
+      w2[2] = __byte_perm (w0[1], w0[2], selector);
+      w2[1] = __byte_perm (w0[0], w0[1], selector);
+      w2[0] = __byte_perm (    0, w0[0], selector);
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 9:
+      w3[1] = __byte_perm (w0[3], w1[0], selector);
+      w3[0] = __byte_perm (w0[2], w0[3], selector);
+      w2[3] = __byte_perm (w0[1], w0[2], selector);
+      w2[2] = __byte_perm (w0[0], w0[1], selector);
+      w2[1] = __byte_perm (    0, w0[0], selector);
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 10:
+      w3[1] = __byte_perm (w0[2], w0[3], selector);
+      w3[0] = __byte_perm (w0[1], w0[2], selector);
+      w2[3] = __byte_perm (w0[0], w0[1], selector);
+      w2[2] = __byte_perm (    0, w0[0], selector);
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 11:
+      w3[1] = __byte_perm (w0[1], w0[2], selector);
+      w3[0] = __byte_perm (w0[0], w0[1], selector);
+      w2[3] = __byte_perm (    0, w0[0], selector);
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 12:
+      w3[1] = __byte_perm (w0[0], w0[1], selector);
+      w3[0] = __byte_perm (    0, w0[0], selector);
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 13:
+      w3[1] = __byte_perm (    0, w0[0], selector);
+      w3[0] = 0;
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+  }
+  #endif
+}
+
+inline void switch_buffer_by_offset_be (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32 offset)
+{
+  #if defined IS_AMD || defined IS_GENERIC
+  switch (offset / 4)
+  {
+    case 0:
+      w3[2] = amd_bytealign (w3[1],     0, offset);
+      w3[1] = amd_bytealign (w3[0], w3[1], offset);
+      w3[0] = amd_bytealign (w2[3], w3[0], offset);
+      w2[3] = amd_bytealign (w2[2], w2[3], offset);
+      w2[2] = amd_bytealign (w2[1], w2[2], offset);
+      w2[1] = amd_bytealign (w2[0], w2[1], offset);
+      w2[0] = amd_bytealign (w1[3], w2[0], offset);
+      w1[3] = amd_bytealign (w1[2], w1[3], offset);
+      w1[2] = amd_bytealign (w1[1], w1[2], offset);
+      w1[1] = amd_bytealign (w1[0], w1[1], offset);
+      w1[0] = amd_bytealign (w0[3], w1[0], offset);
+      w0[3] = amd_bytealign (w0[2], w0[3], offset);
+      w0[2] = amd_bytealign (w0[1], w0[2], offset);
+      w0[1] = amd_bytealign (w0[0], w0[1], offset);
+      w0[0] = amd_bytealign (    0, w0[0], offset);
+      break;
+
+    case 1:
+      w3[2] = amd_bytealign (w3[0],     0, offset);
+      w3[1] = amd_bytealign (w2[3], w3[0], offset);
+      w3[0] = amd_bytealign (w2[2], w2[3], offset);
+      w2[3] = amd_bytealign (w2[1], w2[2], offset);
+      w2[2] = amd_bytealign (w2[0], w2[1], offset);
+      w2[1] = amd_bytealign (w1[3], w2[0], offset);
+      w2[0] = amd_bytealign (w1[2], w1[3], offset);
+      w1[3] = amd_bytealign (w1[1], w1[2], offset);
+      w1[2] = amd_bytealign (w1[0], w1[1], offset);
+      w1[1] = amd_bytealign (w0[3], w1[0], offset);
+      w1[0] = amd_bytealign (w0[2], w0[3], offset);
+      w0[3] = amd_bytealign (w0[1], w0[2], offset);
+      w0[2] = amd_bytealign (w0[0], w0[1], offset);
+      w0[1] = amd_bytealign (    0, w0[0], offset);
+      w0[0] = 0;
+      break;
+
+    case 2:
+      w3[2] = amd_bytealign (w2[3],     0, offset);
+      w3[1] = amd_bytealign (w2[2], w2[3], offset);
+      w3[0] = amd_bytealign (w2[1], w2[2], offset);
+      w2[3] = amd_bytealign (w2[0], w2[1], offset);
+      w2[2] = amd_bytealign (w1[3], w2[0], offset);
+      w2[1] = amd_bytealign (w1[2], w1[3], offset);
+      w2[0] = amd_bytealign (w1[1], w1[2], offset);
+      w1[3] = amd_bytealign (w1[0], w1[1], offset);
+      w1[2] = amd_bytealign (w0[3], w1[0], offset);
+      w1[1] = amd_bytealign (w0[2], w0[3], offset);
+      w1[0] = amd_bytealign (w0[1], w0[2], offset);
+      w0[3] = amd_bytealign (w0[0], w0[1], offset);
+      w0[2] = amd_bytealign (    0, w0[0], offset);
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 3:
+      w3[2] = amd_bytealign (w2[2],     0, offset);
+      w3[1] = amd_bytealign (w2[1], w2[2], offset);
+      w3[0] = amd_bytealign (w2[0], w2[1], offset);
+      w2[3] = amd_bytealign (w1[3], w2[0], offset);
+      w2[2] = amd_bytealign (w1[2], w1[3], offset);
+      w2[1] = amd_bytealign (w1[1], w1[2], offset);
+      w2[0] = amd_bytealign (w1[0], w1[1], offset);
+      w1[3] = amd_bytealign (w0[3], w1[0], offset);
+      w1[2] = amd_bytealign (w0[2], w0[3], offset);
+      w1[1] = amd_bytealign (w0[1], w0[2], offset);
+      w1[0] = amd_bytealign (w0[0], w0[1], offset);
+      w0[3] = amd_bytealign (    0, w0[0], offset);
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 4:
+      w3[2] = amd_bytealign (w2[1],     0, offset);
+      w3[1] = amd_bytealign (w2[0], w2[1], offset);
+      w3[0] = amd_bytealign (w1[3], w2[0], offset);
+      w2[3] = amd_bytealign (w1[2], w1[3], offset);
+      w2[2] = amd_bytealign (w1[1], w1[2], offset);
+      w2[1] = amd_bytealign (w1[0], w1[1], offset);
+      w2[0] = amd_bytealign (w0[3], w1[0], offset);
+      w1[3] = amd_bytealign (w0[2], w0[3], offset);
+      w1[2] = amd_bytealign (w0[1], w0[2], offset);
+      w1[1] = amd_bytealign (w0[0], w0[1], offset);
+      w1[0] = amd_bytealign (    0, w0[0], offset);
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 5:
+      w3[2] = amd_bytealign (w2[0],     0, offset);
+      w3[1] = amd_bytealign (w1[3], w2[0], offset);
+      w3[0] = amd_bytealign (w1[2], w1[3], offset);
+      w2[3] = amd_bytealign (w1[1], w1[2], offset);
+      w2[2] = amd_bytealign (w1[0], w1[1], offset);
+      w2[1] = amd_bytealign (w0[3], w1[0], offset);
+      w2[0] = amd_bytealign (w0[2], w0[3], offset);
+      w1[3] = amd_bytealign (w0[1], w0[2], offset);
+      w1[2] = amd_bytealign (w0[0], w0[1], offset);
+      w1[1] = amd_bytealign (    0, w0[0], offset);
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 6:
+      w3[2] = amd_bytealign (w1[3],     0, offset);
+      w3[1] = amd_bytealign (w1[2], w1[3], offset);
+      w3[0] = amd_bytealign (w1[1], w1[2], offset);
+      w2[3] = amd_bytealign (w1[0], w1[1], offset);
+      w2[2] = amd_bytealign (w0[3], w1[0], offset);
+      w2[1] = amd_bytealign (w0[2], w0[3], offset);
+      w2[0] = amd_bytealign (w0[1], w0[2], offset);
+      w1[3] = amd_bytealign (w0[0], w0[1], offset);
+      w1[2] = amd_bytealign (    0, w0[0], offset);
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 7:
+      w3[2] = amd_bytealign (w1[2],     0, offset);
+      w3[1] = amd_bytealign (w1[1], w1[2], offset);
+      w3[0] = amd_bytealign (w1[0], w1[1], offset);
+      w2[3] = amd_bytealign (w0[3], w1[0], offset);
+      w2[2] = amd_bytealign (w0[2], w0[3], offset);
+      w2[1] = amd_bytealign (w0[1], w0[2], offset);
+      w2[0] = amd_bytealign (w0[0], w0[1], offset);
+      w1[3] = amd_bytealign (    0, w0[0], offset);
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 8:
+      w3[2] = amd_bytealign (w1[1],     0, offset);
+      w3[1] = amd_bytealign (w1[0], w1[1], offset);
+      w3[0] = amd_bytealign (w0[3], w1[0], offset);
+      w2[3] = amd_bytealign (w0[2], w0[3], offset);
+      w2[2] = amd_bytealign (w0[1], w0[2], offset);
+      w2[1] = amd_bytealign (w0[0], w0[1], offset);
+      w2[0] = amd_bytealign (    0, w0[0], offset);
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 9:
+      w3[2] = amd_bytealign (w1[0],     0, offset);
+      w3[1] = amd_bytealign (w0[3], w1[0], offset);
+      w3[0] = amd_bytealign (w0[2], w0[3], offset);
+      w2[3] = amd_bytealign (w0[1], w0[2], offset);
+      w2[2] = amd_bytealign (w0[0], w0[1], offset);
+      w2[1] = amd_bytealign (    0, w0[0], offset);
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 10:
+      w3[2] = amd_bytealign (w0[3],     0, offset);
+      w3[1] = amd_bytealign (w0[2], w0[3], offset);
+      w3[0] = amd_bytealign (w0[1], w0[2], offset);
+      w2[3] = amd_bytealign (w0[0], w0[1], offset);
+      w2[2] = amd_bytealign (    0, w0[0], offset);
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 11:
+      w3[2] = amd_bytealign (w0[2],     0, offset);
+      w3[1] = amd_bytealign (w0[1], w0[2], offset);
+      w3[0] = amd_bytealign (w0[0], w0[1], offset);
+      w2[3] = amd_bytealign (    0, w0[0], offset);
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 12:
+      w3[2] = amd_bytealign (w0[1],     0, offset);
+      w3[1] = amd_bytealign (w0[0], w0[1], offset);
+      w3[0] = amd_bytealign (    0, w0[0], offset);
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 13:
+      w3[2] = amd_bytealign (w0[0],     0, offset);
+      w3[1] = amd_bytealign (    0, w0[0], offset);
+      w3[0] = 0;
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+  }
+  #endif
+
+  #ifdef IS_NV
+  const int selector = (0x76543210 >> ((offset & 3) * 4)) & 0xffff;
+
+  switch (offset / 4)
+  {
+    case 0:
+      w3[1] = __byte_perm (w3[1], w3[0], selector);
+      w3[0] = __byte_perm (w3[0], w2[3], selector);
+      w2[3] = __byte_perm (w2[3], w2[2], selector);
+      w2[2] = __byte_perm (w2[2], w2[1], selector);
+      w2[1] = __byte_perm (w2[1], w2[0], selector);
+      w2[0] = __byte_perm (w2[0], w1[3], selector);
+      w1[3] = __byte_perm (w1[3], w1[2], selector);
+      w1[2] = __byte_perm (w1[2], w1[1], selector);
+      w1[1] = __byte_perm (w1[1], w1[0], selector);
+      w1[0] = __byte_perm (w1[0], w0[3], selector);
+      w0[3] = __byte_perm (w0[3], w0[2], selector);
+      w0[2] = __byte_perm (w0[2], w0[1], selector);
+      w0[1] = __byte_perm (w0[1], w0[0], selector);
+      w0[0] = __byte_perm (w0[0],     0, selector);
+      break;
+
+    case 1:
+      w3[1] = __byte_perm (w3[0], w2[3], selector);
+      w3[0] = __byte_perm (w2[3], w2[2], selector);
+      w2[3] = __byte_perm (w2[2], w2[1], selector);
+      w2[2] = __byte_perm (w2[1], w2[0], selector);
+      w2[1] = __byte_perm (w2[0], w1[3], selector);
+      w2[0] = __byte_perm (w1[3], w1[2], selector);
+      w1[3] = __byte_perm (w1[2], w1[1], selector);
+      w1[2] = __byte_perm (w1[1], w1[0], selector);
+      w1[1] = __byte_perm (w1[0], w0[3], selector);
+      w1[0] = __byte_perm (w0[3], w0[2], selector);
+      w0[3] = __byte_perm (w0[2], w0[1], selector);
+      w0[2] = __byte_perm (w0[1], w0[0], selector);
+      w0[1] = __byte_perm (w0[0],     0, selector);
+      w0[0] = 0;
+      break;
+
+    case 2:
+      w3[1] = __byte_perm (w2[3], w2[2], selector);
+      w3[0] = __byte_perm (w2[2], w2[1], selector);
+      w2[3] = __byte_perm (w2[1], w2[0], selector);
+      w2[2] = __byte_perm (w2[0], w1[3], selector);
+      w2[1] = __byte_perm (w1[3], w1[2], selector);
+      w2[0] = __byte_perm (w1[2], w1[1], selector);
+      w1[3] = __byte_perm (w1[1], w1[0], selector);
+      w1[2] = __byte_perm (w1[0], w0[3], selector);
+      w1[1] = __byte_perm (w0[3], w0[2], selector);
+      w1[0] = __byte_perm (w0[2], w0[1], selector);
+      w0[3] = __byte_perm (w0[1], w0[0], selector);
+      w0[2] = __byte_perm (w0[0],     0, selector);
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 3:
+      w3[1] = __byte_perm (w2[2], w2[1], selector);
+      w3[0] = __byte_perm (w2[1], w2[0], selector);
+      w2[3] = __byte_perm (w2[0], w1[3], selector);
+      w2[2] = __byte_perm (w1[3], w1[2], selector);
+      w2[1] = __byte_perm (w1[2], w1[1], selector);
+      w2[0] = __byte_perm (w1[1], w1[0], selector);
+      w1[3] = __byte_perm (w1[0], w0[3], selector);
+      w1[2] = __byte_perm (w0[3], w0[2], selector);
+      w1[1] = __byte_perm (w0[2], w0[1], selector);
+      w1[0] = __byte_perm (w0[1], w0[0], selector);
+      w0[3] = __byte_perm (w0[0],     0, selector);
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 4:
+      w3[1] = __byte_perm (w2[1], w2[0], selector);
+      w3[0] = __byte_perm (w2[0], w1[3], selector);
+      w2[3] = __byte_perm (w1[3], w1[2], selector);
+      w2[2] = __byte_perm (w1[2], w1[1], selector);
+      w2[1] = __byte_perm (w1[1], w1[0], selector);
+      w2[0] = __byte_perm (w1[0], w0[3], selector);
+      w1[3] = __byte_perm (w0[3], w0[2], selector);
+      w1[2] = __byte_perm (w0[2], w0[1], selector);
+      w1[1] = __byte_perm (w0[1], w0[0], selector);
+      w1[0] = __byte_perm (w0[0],     0, selector);
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 5:
+      w3[1] = __byte_perm (w2[0], w1[3], selector);
+      w3[0] = __byte_perm (w1[3], w1[2], selector);
+      w2[3] = __byte_perm (w1[2], w1[1], selector);
+      w2[2] = __byte_perm (w1[1], w1[0], selector);
+      w2[1] = __byte_perm (w1[0], w0[3], selector);
+      w2[0] = __byte_perm (w0[3], w0[2], selector);
+      w1[3] = __byte_perm (w0[2], w0[1], selector);
+      w1[2] = __byte_perm (w0[1], w0[0], selector);
+      w1[1] = __byte_perm (w0[0],     0, selector);
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 6:
+      w3[1] = __byte_perm (w1[3], w1[2], selector);
+      w3[0] = __byte_perm (w1[2], w1[1], selector);
+      w2[3] = __byte_perm (w1[1], w1[0], selector);
+      w2[2] = __byte_perm (w1[0], w0[3], selector);
+      w2[1] = __byte_perm (w0[3], w0[2], selector);
+      w2[0] = __byte_perm (w0[2], w0[1], selector);
+      w1[3] = __byte_perm (w0[1], w0[0], selector);
+      w1[2] = __byte_perm (w0[0],     0, selector);
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 7:
+      w3[1] = __byte_perm (w1[2], w1[1], selector);
+      w3[0] = __byte_perm (w1[1], w1[0], selector);
+      w2[3] = __byte_perm (w1[0], w0[3], selector);
+      w2[2] = __byte_perm (w0[3], w0[2], selector);
+      w2[1] = __byte_perm (w0[2], w0[1], selector);
+      w2[0] = __byte_perm (w0[1], w0[0], selector);
+      w1[3] = __byte_perm (w0[0],     0, selector);
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 8:
+      w3[1] = __byte_perm (w1[1], w1[0], selector);
+      w3[0] = __byte_perm (w1[0], w0[3], selector);
+      w2[3] = __byte_perm (w0[3], w0[2], selector);
+      w2[2] = __byte_perm (w0[2], w0[1], selector);
+      w2[1] = __byte_perm (w0[1], w0[0], selector);
+      w2[0] = __byte_perm (w0[0],     0, selector);
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 9:
+      w3[1] = __byte_perm (w1[0], w0[3], selector);
+      w3[0] = __byte_perm (w0[3], w0[2], selector);
+      w2[3] = __byte_perm (w0[2], w0[1], selector);
+      w2[2] = __byte_perm (w0[1], w0[0], selector);
+      w2[1] = __byte_perm (w0[0],     0, selector);
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 10:
+      w3[1] = __byte_perm (w0[3], w0[2], selector);
+      w3[0] = __byte_perm (w0[2], w0[1], selector);
+      w2[3] = __byte_perm (w0[1], w0[0], selector);
+      w2[2] = __byte_perm (w0[0],     0, selector);
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 11:
+      w3[1] = __byte_perm (w0[2], w0[1], selector);
+      w3[0] = __byte_perm (w0[1], w0[0], selector);
+      w2[3] = __byte_perm (w0[0],     0, selector);
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 12:
+      w3[1] = __byte_perm (w0[1], w0[0], selector);
+      w3[0] = __byte_perm (w0[0],     0, selector);
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 13:
+      w3[1] = __byte_perm (w0[0],     0, selector);
+      w3[0] = 0;
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+  }
+  #endif
+}
+
+inline void overwrite_at_le (u32x sw[16], const u32x w0, const u32 salt_len)
+{
+  #if defined cl_amd_media_ops
+  switch (salt_len)
+  {
+    case  0:  sw[0] = w0;
+              break;
+    case  1:  sw[0] = amd_bytealign (w0, sw[0] << 24, 3);
+              sw[1] = amd_bytealign (sw[1] >>  8, w0, 3);
+              break;
+    case  2:  sw[0] = amd_bytealign (w0, sw[0] << 16, 2);
+              sw[1] = amd_bytealign (sw[1] >> 16, w0, 2);
+              break;
+    case  3:  sw[0] = amd_bytealign (w0, sw[0] <<  8, 1);
+              sw[1] = amd_bytealign (sw[1] >> 24, w0, 1);
+              break;
+    case  4:  sw[1] = w0;
+              break;
+    case  5:  sw[1] = amd_bytealign (w0, sw[1] << 24, 3);
+              sw[2] = amd_bytealign (sw[2] >>  8, w0, 3);
+              break;
+    case  6:  sw[1] = amd_bytealign (w0, sw[1] << 16, 2);
+              sw[2] = amd_bytealign (sw[2] >> 16, w0, 2);
+              break;
+    case  7:  sw[1] = amd_bytealign (w0, sw[1] <<  8, 1);
+              sw[2] = amd_bytealign (sw[2] >> 24, w0, 1);
+              break;
+    case  8:  sw[2] = w0;
+              break;
+    case  9:  sw[2] = amd_bytealign (w0, sw[2] << 24, 3);
+              sw[3] = amd_bytealign (sw[3] >>  8, w0, 3);
+              break;
+    case 10:  sw[2] = amd_bytealign (w0, sw[2] << 16, 2);
+              sw[3] = amd_bytealign (sw[3] >> 16, w0, 2);
+              break;
+    case 11:  sw[2] = amd_bytealign (w0, sw[2] <<  8, 1);
+              sw[3] = amd_bytealign (sw[3] >> 24, w0, 1);
+              break;
+    case 12:  sw[3] = w0;
+              break;
+    case 13:  sw[3] = amd_bytealign (w0, sw[3] << 24, 3);
+              sw[4] = amd_bytealign (sw[4] >>  8, w0, 3);
+              break;
+    case 14:  sw[3] = amd_bytealign (w0, sw[3] << 16, 2);
+              sw[4] = amd_bytealign (sw[4] >> 16, w0, 2);
+              break;
+    case 15:  sw[3] = amd_bytealign (w0, sw[3] <<  8, 1);
+              sw[4] = amd_bytealign (sw[4] >> 24, w0, 1);
+              break;
+    case 16:  sw[4] = w0;
+              break;
+    case 17:  sw[4] = amd_bytealign (w0, sw[4] << 24, 3);
+              sw[5] = amd_bytealign (sw[5] >>  8, w0, 3);
+              break;
+    case 18:  sw[4] = amd_bytealign (w0, sw[4] << 16, 2);
+              sw[5] = amd_bytealign (sw[5] >> 16, w0, 2);
+              break;
+    case 19:  sw[4] = amd_bytealign (w0, sw[4] <<  8, 1);
+              sw[5] = amd_bytealign (sw[5] >> 24, w0, 1);
+              break;
+    case 20:  sw[5] = w0;
+              break;
+    case 21:  sw[5] = amd_bytealign (w0, sw[5] << 24, 3);
+              sw[6] = amd_bytealign (sw[6] >>  8, w0, 3);
+              break;
+    case 22:  sw[5] = amd_bytealign (w0, sw[5] << 16, 2);
+              sw[6] = amd_bytealign (sw[6] >> 16, w0, 2);
+              break;
+    case 23:  sw[5] = amd_bytealign (w0, sw[5] <<  8, 1);
+              sw[6] = amd_bytealign (sw[6] >> 24, w0, 1);
+              break;
+    case 24:  sw[6] = w0;
+              break;
+    case 25:  sw[6] = amd_bytealign (w0, sw[6] << 24, 3);
+              sw[7] = amd_bytealign (sw[7] >>  8, w0, 3);
+              break;
+    case 26:  sw[6] = amd_bytealign (w0, sw[6] << 16, 2);
+              sw[7] = amd_bytealign (sw[7] >> 16, w0, 2);
+              break;
+    case 27:  sw[6] = amd_bytealign (w0, sw[6] <<  8, 1);
+              sw[7] = amd_bytealign (sw[7] >> 24, w0, 1);
+              break;
+    case 28:  sw[7] = w0;
+              break;
+    case 29:  sw[7] = amd_bytealign (w0, sw[7] << 24, 3);
+              sw[8] = amd_bytealign (sw[8] >>  8, w0, 3);
+              break;
+    case 30:  sw[7] = amd_bytealign (w0, sw[7] << 16, 2);
+              sw[8] = amd_bytealign (sw[8] >> 16, w0, 2);
+              break;
+    case 31:  sw[7] = amd_bytealign (w0, sw[7] <<  8, 1);
+              sw[8] = amd_bytealign (sw[8] >> 24, w0, 1);
+              break;
+  }
+  #else
+  switch (salt_len)
+  {
+    case  0:  sw[0] =  w0;
+              break;
+    case  1:  sw[0] = (sw[0] & 0x000000ff) | (w0 <<  8);
+              sw[1] = (sw[1] & 0xffffff00) | (w0 >> 24);
+              break;
+    case  2:  sw[0] = (sw[0] & 0x0000ffff) | (w0 << 16);
+              sw[1] = (sw[1] & 0xffff0000) | (w0 >> 16);
+              break;
+    case  3:  sw[0] = (sw[0] & 0x00ffffff) | (w0 << 24);
+              sw[1] = (sw[1] & 0xff000000) | (w0 >>  8);
+              break;
+    case  4:  sw[1] =  w0;
+              break;
+    case  5:  sw[1] = (sw[1] & 0x000000ff) | (w0 <<  8);
+              sw[2] = (sw[2] & 0xffffff00) | (w0 >> 24);
+              break;
+    case  6:  sw[1] = (sw[1] & 0x0000ffff) | (w0 << 16);
+              sw[2] = (sw[2] & 0xffff0000) | (w0 >> 16);
+              break;
+    case  7:  sw[1] = (sw[1] & 0x00ffffff) | (w0 << 24);
+              sw[2] = (sw[2] & 0xff000000) | (w0 >>  8);
+              break;
+    case  8:  sw[2] =  w0;
+              break;
+    case  9:  sw[2] = (sw[2] & 0x000000ff) | (w0 <<  8);
+              sw[3] = (sw[3] & 0xffffff00) | (w0 >> 24);
+              break;
+    case 10:  sw[2] = (sw[2] & 0x0000ffff) | (w0 << 16);
+              sw[3] = (sw[3] & 0xffff0000) | (w0 >> 16);
+              break;
+    case 11:  sw[2] = (sw[2] & 0x00ffffff) | (w0 << 24);
+              sw[3] = (sw[3] & 0xff000000) | (w0 >>  8);
+              break;
+    case 12:  sw[3] =  w0;
+              break;
+    case 13:  sw[3] = (sw[3] & 0x000000ff) | (w0 <<  8);
+              sw[4] = (sw[4] & 0xffffff00) | (w0 >> 24);
+              break;
+    case 14:  sw[3] = (sw[3] & 0x0000ffff) | (w0 << 16);
+              sw[4] = (sw[4] & 0xffff0000) | (w0 >> 16);
+              break;
+    case 15:  sw[3] = (sw[3] & 0x00ffffff) | (w0 << 24);
+              sw[4] = (sw[4] & 0xff000000) | (w0 >>  8);
+              break;
+    case 16:  sw[4] =  w0;
+              break;
+    case 17:  sw[4] = (sw[4] & 0x000000ff) | (w0 <<  8);
+              sw[5] = (sw[5] & 0xffffff00) | (w0 >> 24);
+              break;
+    case 18:  sw[4] = (sw[4] & 0x0000ffff) | (w0 << 16);
+              sw[5] = (sw[5] & 0xffff0000) | (w0 >> 16);
+              break;
+    case 19:  sw[4] = (sw[4] & 0x00ffffff) | (w0 << 24);
+              sw[5] = (sw[5] & 0xff000000) | (w0 >>  8);
+              break;
+    case 20:  sw[5] =  w0;
+              break;
+    case 21:  sw[5] = (sw[5] & 0x000000ff) | (w0 <<  8);
+              sw[6] = (sw[6] & 0xffffff00) | (w0 >> 24);
+              break;
+    case 22:  sw[5] = (sw[5] & 0x0000ffff) | (w0 << 16);
+              sw[6] = (sw[6] & 0xffff0000) | (w0 >> 16);
+              break;
+    case 23:  sw[5] = (sw[5] & 0x00ffffff) | (w0 << 24);
+              sw[6] = (sw[6] & 0xff000000) | (w0 >>  8);
+              break;
+    case 24:  sw[6] =  w0;
+              break;
+    case 25:  sw[6] = (sw[6] & 0x000000ff) | (w0 <<  8);
+              sw[7] = (sw[7] & 0xffffff00) | (w0 >> 24);
+              break;
+    case 26:  sw[6] = (sw[6] & 0x0000ffff) | (w0 << 16);
+              sw[7] = (sw[7] & 0xffff0000) | (w0 >> 16);
+              break;
+    case 27:  sw[6] = (sw[6] & 0x00ffffff) | (w0 << 24);
+              sw[7] = (sw[7] & 0xff000000) | (w0 >>  8);
+              break;
+    case 28:  sw[7] =  w0;
+              break;
+    case 29:  sw[7] = (sw[7] & 0x000000ff) | (w0 <<  8);
+              sw[8] = (sw[8] & 0xffffff00) | (w0 >> 24);
+              break;
+    case 30:  sw[7] = (sw[7] & 0x0000ffff) | (w0 << 16);
+              sw[8] = (sw[8] & 0xffff0000) | (w0 >> 16);
+              break;
+    case 31:  sw[7] = (sw[7] & 0x00ffffff) | (w0 << 24);
+              sw[8] = (sw[8] & 0xff000000) | (w0 >>  8);
+              break;
+  }
+  #endif
+}
+
+inline void overwrite_at_be (u32x sw[16], const u32x w0, const u32 salt_len)
+{
+  // would be nice to have optimization based on amd_bytealign as with _le counterpart
+
+  switch (salt_len)
+  {
+    case  0:  sw[0] =  w0;
+              break;
+    case  1:  sw[0] = (sw[0] & 0xff000000) | (w0 >>  8);
+              sw[1] = (sw[1] & 0x00ffffff) | (w0 << 24);
+              break;
+    case  2:  sw[0] = (sw[0] & 0xffff0000) | (w0 >> 16);
+              sw[1] = (sw[1] & 0x0000ffff) | (w0 << 16);
+              break;
+    case  3:  sw[0] = (sw[0] & 0xffffff00) | (w0 >> 24);
+              sw[1] = (sw[1] & 0x000000ff) | (w0 <<  8);
+              break;
+    case  4:  sw[1] =  w0;
+              break;
+    case  5:  sw[1] = (sw[1] & 0xff000000) | (w0 >>  8);
+              sw[2] = (sw[2] & 0x00ffffff) | (w0 << 24);
+              break;
+    case  6:  sw[1] = (sw[1] & 0xffff0000) | (w0 >> 16);
+              sw[2] = (sw[2] & 0x0000ffff) | (w0 << 16);
+              break;
+    case  7:  sw[1] = (sw[1] & 0xffffff00) | (w0 >> 24);
+              sw[2] = (sw[2] & 0x000000ff) | (w0 <<  8);
+              break;
+    case  8:  sw[2] =  w0;
+              break;
+    case  9:  sw[2] = (sw[2] & 0xff000000) | (w0 >>  8);
+              sw[3] = (sw[3] & 0x00ffffff) | (w0 << 24);
+              break;
+    case 10:  sw[2] = (sw[2] & 0xffff0000) | (w0 >> 16);
+              sw[3] = (sw[3] & 0x0000ffff) | (w0 << 16);
+              break;
+    case 11:  sw[2] = (sw[2] & 0xffffff00) | (w0 >> 24);
+              sw[3] = (sw[3] & 0x000000ff) | (w0 <<  8);
+              break;
+    case 12:  sw[3] =  w0;
+              break;
+    case 13:  sw[3] = (sw[3] & 0xff000000) | (w0 >>  8);
+              sw[4] = (sw[4] & 0x00ffffff) | (w0 << 24);
+              break;
+    case 14:  sw[3] = (sw[3] & 0xffff0000) | (w0 >> 16);
+              sw[4] = (sw[4] & 0x0000ffff) | (w0 << 16);
+              break;
+    case 15:  sw[3] = (sw[3] & 0xffffff00) | (w0 >> 24);
+              sw[4] = (sw[4] & 0x000000ff) | (w0 <<  8);
+              break;
+    case 16:  sw[4] =  w0;
+              break;
+    case 17:  sw[4] = (sw[4] & 0xff000000) | (w0 >>  8);
+              sw[5] = (sw[5] & 0x00ffffff) | (w0 << 24);
+              break;
+    case 18:  sw[4] = (sw[4] & 0xffff0000) | (w0 >> 16);
+              sw[5] = (sw[5] & 0x0000ffff) | (w0 << 16);
+              break;
+    case 19:  sw[4] = (sw[4] & 0xffffff00) | (w0 >> 24);
+              sw[5] = (sw[5] & 0x000000ff) | (w0 <<  8);
+              break;
+    case 20:  sw[5] =  w0;
+              break;
+    case 21:  sw[5] = (sw[5] & 0xff000000) | (w0 >>  8);
+              sw[6] = (sw[6] & 0x00ffffff) | (w0 << 24);
+              break;
+    case 22:  sw[5] = (sw[5] & 0xffff0000) | (w0 >> 16);
+              sw[6] = (sw[6] & 0x0000ffff) | (w0 << 16);
+              break;
+    case 23:  sw[5] = (sw[5] & 0xffffff00) | (w0 >> 24);
+              sw[6] = (sw[6] & 0x000000ff) | (w0 <<  8);
+              break;
+    case 24:  sw[6] =  w0;
+              break;
+    case 25:  sw[6] = (sw[6] & 0xff000000) | (w0 >>  8);
+              sw[7] = (sw[7] & 0x00ffffff) | (w0 << 24);
+              break;
+    case 26:  sw[6] = (sw[6] & 0xffff0000) | (w0 >> 16);
+              sw[7] = (sw[7] & 0x0000ffff) | (w0 << 16);
+              break;
+    case 27:  sw[6] = (sw[6] & 0xffffff00) | (w0 >> 24);
+              sw[7] = (sw[7] & 0x000000ff) | (w0 <<  8);
+              break;
+    case 28:  sw[7] =  w0;
+              break;
+    case 29:  sw[7] = (sw[7] & 0xff000000) | (w0 >>  8);
+              sw[8] = (sw[8] & 0x00ffffff) | (w0 << 24);
+              break;
+    case 30:  sw[7] = (sw[7] & 0xffff0000) | (w0 >> 16);
+              sw[8] = (sw[8] & 0x0000ffff) | (w0 << 16);
+              break;
+    case 31:  sw[7] = (sw[7] & 0xffffff00) | (w0 >> 24);
+              sw[8] = (sw[8] & 0x000000ff) | (w0 <<  8);
+              break;
+  }
+}
+
+inline void overwrite_at_le_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x wx, const u32 salt_len)
+{
+  #if defined cl_amd_media_ops
+  switch (salt_len)
+  {
+    case  0:  w0[0] = wx;
+              break;
+    case  1:  w0[0] = amd_bytealign (wx, w0[0] << 24, 3);
+              w0[1] = amd_bytealign (w0[1] >>  8, wx, 3);
+              break;
+    case  2:  w0[0] = amd_bytealign (wx, w0[0] << 16, 2);
+              w0[1] = amd_bytealign (w0[1] >> 16, wx, 2);
+              break;
+    case  3:  w0[0] = amd_bytealign (wx, w0[0] <<  8, 1);
+              w0[1] = amd_bytealign (w0[1] >> 24, wx, 1);
+              break;
+    case  4:  w0[1] = wx;
+              break;
+    case  5:  w0[1] = amd_bytealign (wx, w0[1] << 24, 3);
+              w0[2] = amd_bytealign (w0[2] >>  8, wx, 3);
+              break;
+    case  6:  w0[1] = amd_bytealign (wx, w0[1] << 16, 2);
+              w0[2] = amd_bytealign (w0[2] >> 16, wx, 2);
+              break;
+    case  7:  w0[1] = amd_bytealign (wx, w0[1] <<  8, 1);
+              w0[2] = amd_bytealign (w0[2] >> 24, wx, 1);
+              break;
+    case  8:  w0[2] = wx;
+              break;
+    case  9:  w0[2] = amd_bytealign (wx, w0[2] << 24, 3);
+              w0[3] = amd_bytealign (w0[3] >>  8, wx, 3);
+              break;
+    case 10:  w0[2] = amd_bytealign (wx, w0[2] << 16, 2);
+              w0[3] = amd_bytealign (w0[3] >> 16, wx, 2);
+              break;
+    case 11:  w0[2] = amd_bytealign (wx, w0[2] <<  8, 1);
+              w0[3] = amd_bytealign (w0[3] >> 24, wx, 1);
+              break;
+    case 12:  w0[3] = wx;
+              break;
+    case 13:  w0[3] = amd_bytealign (wx, w0[3] << 24, 3);
+              w1[0] = amd_bytealign (w1[0] >>  8, wx, 3);
+              break;
+    case 14:  w0[3] = amd_bytealign (wx, w0[3] << 16, 2);
+              w1[0] = amd_bytealign (w1[0] >> 16, wx, 2);
+              break;
+    case 15:  w0[3] = amd_bytealign (wx, w0[3] <<  8, 1);
+              w1[0] = amd_bytealign (w1[0] >> 24, wx, 1);
+              break;
+    case 16:  w1[0] = wx;
+              break;
+    case 17:  w1[0] = amd_bytealign (wx, w1[0] << 24, 3);
+              w1[1] = amd_bytealign (w1[1] >>  8, wx, 3);
+              break;
+    case 18:  w1[0] = amd_bytealign (wx, w1[0] << 16, 2);
+              w1[1] = amd_bytealign (w1[1] >> 16, wx, 2);
+              break;
+    case 19:  w1[0] = amd_bytealign (wx, w1[0] <<  8, 1);
+              w1[1] = amd_bytealign (w1[1] >> 24, wx, 1);
+              break;
+    case 20:  w1[1] = wx;
+              break;
+    case 21:  w1[1] = amd_bytealign (wx, w1[1] << 24, 3);
+              w1[2] = amd_bytealign (w1[2] >>  8, wx, 3);
+              break;
+    case 22:  w1[1] = amd_bytealign (wx, w1[1] << 16, 2);
+              w1[2] = amd_bytealign (w1[2] >> 16, wx, 2);
+              break;
+    case 23:  w1[1] = amd_bytealign (wx, w1[1] <<  8, 1);
+              w1[2] = amd_bytealign (w1[2] >> 24, wx, 1);
+              break;
+    case 24:  w1[2] = wx;
+              break;
+    case 25:  w1[2] = amd_bytealign (wx, w1[2] << 24, 3);
+              w1[3] = amd_bytealign (w1[3] >>  8, wx, 3);
+              break;
+    case 26:  w1[2] = amd_bytealign (wx, w1[2] << 16, 2);
+              w1[3] = amd_bytealign (w1[3] >> 16, wx, 2);
+              break;
+    case 27:  w1[2] = amd_bytealign (wx, w1[2] <<  8, 1);
+              w1[3] = amd_bytealign (w1[3] >> 24, wx, 1);
+              break;
+    case 28:  w1[3] = wx;
+              break;
+    case 29:  w1[3] = amd_bytealign (wx, w1[3] << 24, 3);
+              w2[0] = amd_bytealign (w2[0] >>  8, wx, 3);
+              break;
+    case 30:  w1[3] = amd_bytealign (wx, w1[3] << 16, 2);
+              w2[0] = amd_bytealign (w2[0] >> 16, wx, 2);
+              break;
+    case 31:  w1[3] = amd_bytealign (wx, w1[3] <<  8, 1);
+              w2[0] = amd_bytealign (w2[0] >> 24, wx, 1);
+              break;
+    case 32:  w2[0] = wx;
+              break;
+    case 33:  w2[0] = amd_bytealign (wx, w2[0] << 24, 3);
+              w2[1] = amd_bytealign (w2[1] >>  8, wx, 3);
+              break;
+    case 34:  w2[0] = amd_bytealign (wx, w2[0] << 16, 2);
+              w2[1] = amd_bytealign (w2[1] >> 16, wx, 2);
+              break;
+    case 35:  w2[0] = amd_bytealign (wx, w2[0] <<  8, 1);
+              w2[1] = amd_bytealign (w2[1] >> 24, wx, 1);
+              break;
+    case 36:  w2[1] = wx;
+              break;
+    case 37:  w2[1] = amd_bytealign (wx, w2[1] << 24, 3);
+              w2[2] = amd_bytealign (w2[2] >>  8, wx, 3);
+              break;
+    case 38:  w2[1] = amd_bytealign (wx, w2[1] << 16, 2);
+              w2[2] = amd_bytealign (w2[2] >> 16, wx, 2);
+              break;
+    case 39:  w2[1] = amd_bytealign (wx, w2[1] <<  8, 1);
+              w2[2] = amd_bytealign (w2[2] >> 24, wx, 1);
+              break;
+    case 40:  w2[2] = wx;
+              break;
+    case 41:  w2[2] = amd_bytealign (wx, w2[2] << 24, 3);
+              w2[3] = amd_bytealign (w2[3] >>  8, wx, 3);
+              break;
+    case 42:  w2[2] = amd_bytealign (wx, w2[2] << 16, 2);
+              w2[3] = amd_bytealign (w2[3] >> 16, wx, 2);
+              break;
+    case 43:  w2[2] = amd_bytealign (wx, w2[2] <<  8, 1);
+              w2[3] = amd_bytealign (w2[3] >> 24, wx, 1);
+              break;
+    case 44:  w2[3] = wx;
+              break;
+    case 45:  w2[3] = amd_bytealign (wx, w2[3] << 24, 3);
+              w3[0] = amd_bytealign (w3[0] >>  8, wx, 3);
+              break;
+    case 46:  w2[3] = amd_bytealign (wx, w2[3] << 16, 2);
+              w3[0] = amd_bytealign (w3[0] >> 16, wx, 2);
+              break;
+    case 47:  w2[3] = amd_bytealign (wx, w2[3] <<  8, 1);
+              w3[0] = amd_bytealign (w3[0] >> 24, wx, 1);
+              break;
+    case 48:  w3[0] = wx;
+              break;
+    case 49:  w3[0] = amd_bytealign (wx, w3[0] << 24, 3);
+              w3[1] = amd_bytealign (w3[1] >>  8, wx, 3);
+              break;
+    case 50:  w3[0] = amd_bytealign (wx, w3[0] << 16, 2);
+              w3[1] = amd_bytealign (w3[1] >> 16, wx, 2);
+              break;
+    case 51:  w3[0] = amd_bytealign (wx, w3[0] <<  8, 1);
+              w3[1] = amd_bytealign (w3[1] >> 24, wx, 1);
+              break;
+    case 52:  w3[1] = wx;
+              break;
+    case 53:  w3[1] = amd_bytealign (wx, w3[1] << 24, 3);
+              w3[2] = amd_bytealign (w3[2] >>  8, wx, 3);
+              break;
+    case 54:  w3[1] = amd_bytealign (wx, w3[1] << 16, 2);
+              w3[2] = amd_bytealign (w3[2] >> 16, wx, 2);
+              break;
+    case 55:  w3[1] = amd_bytealign (wx, w3[1] <<  8, 1);
+              w3[2] = amd_bytealign (w3[2] >> 24, wx, 1);
+              break;
+    case 56:  w3[2] = wx;
+              break;
+    case 57:  w3[2] = amd_bytealign (wx, w3[2] << 24, 3);
+              w3[3] = amd_bytealign (w3[3] >>  8, wx, 3);
+              break;
+    case 58:  w3[2] = amd_bytealign (wx, w3[2] << 16, 2);
+              w3[3] = amd_bytealign (w3[3] >> 16, wx, 2);
+              break;
+    case 59:  w3[2] = amd_bytealign (wx, w3[2] <<  8, 1);
+              w3[3] = amd_bytealign (w3[3] >> 24, wx, 1);
+              break;
+    case 60:  w3[3] = wx;
+              break;
+    case 61:  w3[3] = amd_bytealign (wx, w3[3] << 24, 3);
+              //w4[0] = amd_bytealign (w4[0] >>  8, wx, 3);
+              break;
+    case 62:  w3[3] = amd_bytealign (wx, w3[3] << 16, 2);
+              //w4[0] = amd_bytealign (w4[0] >> 16, wx, 2);
+              break;
+    case 63:  w3[3] = amd_bytealign (wx, w3[3] <<  8, 1);
+              //w4[0] = amd_bytealign (w4[0] >> 24, wx, 1);
+              break;
+  }
+  #else
+  switch (salt_len)
+  {
+    case  0:  w0[0] =  wx;
+              break;
+    case  1:  w0[0] = (w0[0] & 0x000000ff) | (wx <<  8);
+              w0[1] = (w0[1] & 0xffffff00) | (wx >> 24);
+              break;
+    case  2:  w0[0] = (w0[0] & 0x0000ffff) | (wx << 16);
+              w0[1] = (w0[1] & 0xffff0000) | (wx >> 16);
+              break;
+    case  3:  w0[0] = (w0[0] & 0x00ffffff) | (wx << 24);
+              w0[1] = (w0[1] & 0xff000000) | (wx >>  8);
+              break;
+    case  4:  w0[1] =  wx;
+              break;
+    case  5:  w0[1] = (w0[1] & 0x000000ff) | (wx <<  8);
+              w0[2] = (w0[2] & 0xffffff00) | (wx >> 24);
+              break;
+    case  6:  w0[1] = (w0[1] & 0x0000ffff) | (wx << 16);
+              w0[2] = (w0[2] & 0xffff0000) | (wx >> 16);
+              break;
+    case  7:  w0[1] = (w0[1] & 0x00ffffff) | (wx << 24);
+              w0[2] = (w0[2] & 0xff000000) | (wx >>  8);
+              break;
+    case  8:  w0[2] =  wx;
+              break;
+    case  9:  w0[2] = (w0[2] & 0x000000ff) | (wx <<  8);
+              w0[3] = (w0[3] & 0xffffff00) | (wx >> 24);
+              break;
+    case 10:  w0[2] = (w0[2] & 0x0000ffff) | (wx << 16);
+              w0[3] = (w0[3] & 0xffff0000) | (wx >> 16);
+              break;
+    case 11:  w0[2] = (w0[2] & 0x00ffffff) | (wx << 24);
+              w0[3] = (w0[3] & 0xff000000) | (wx >>  8);
+              break;
+    case 12:  w0[3] =  wx;
+              break;
+    case 13:  w0[3] = (w0[3] & 0x000000ff) | (wx <<  8);
+              w1[0] = (w1[0] & 0xffffff00) | (wx >> 24);
+              break;
+    case 14:  w0[3] = (w0[3] & 0x0000ffff) | (wx << 16);
+              w1[0] = (w1[0] & 0xffff0000) | (wx >> 16);
+              break;
+    case 15:  w0[3] = (w0[3] & 0x00ffffff) | (wx << 24);
+              w1[0] = (w1[0] & 0xff000000) | (wx >>  8);
+              break;
+    case 16:  w1[0] =  wx;
+              break;
+    case 17:  w1[0] = (w1[0] & 0x000000ff) | (wx <<  8);
+              w1[1] = (w1[1] & 0xffffff00) | (wx >> 24);
+              break;
+    case 18:  w1[0] = (w1[0] & 0x0000ffff) | (wx << 16);
+              w1[1] = (w1[1] & 0xffff0000) | (wx >> 16);
+              break;
+    case 19:  w1[0] = (w1[0] & 0x00ffffff) | (wx << 24);
+              w1[1] = (w1[1] & 0xff000000) | (wx >>  8);
+              break;
+    case 20:  w1[1] =  wx;
+              break;
+    case 21:  w1[1] = (w1[1] & 0x000000ff) | (wx <<  8);
+              w1[2] = (w1[2] & 0xffffff00) | (wx >> 24);
+              break;
+    case 22:  w1[1] = (w1[1] & 0x0000ffff) | (wx << 16);
+              w1[2] = (w1[2] & 0xffff0000) | (wx >> 16);
+              break;
+    case 23:  w1[1] = (w1[1] & 0x00ffffff) | (wx << 24);
+              w1[2] = (w1[2] & 0xff000000) | (wx >>  8);
+              break;
+    case 24:  w1[2] =  wx;
+              break;
+    case 25:  w1[2] = (w1[2] & 0x000000ff) | (wx <<  8);
+              w1[3] = (w1[3] & 0xffffff00) | (wx >> 24);
+              break;
+    case 26:  w1[2] = (w1[2] & 0x0000ffff) | (wx << 16);
+              w1[3] = (w1[3] & 0xffff0000) | (wx >> 16);
+              break;
+    case 27:  w1[2] = (w1[2] & 0x00ffffff) | (wx << 24);
+              w1[3] = (w1[3] & 0xff000000) | (wx >>  8);
+              break;
+    case 28:  w1[3] =  wx;
+              break;
+    case 29:  w1[3] = (w1[3] & 0x000000ff) | (wx <<  8);
+              w2[0] = (w2[0] & 0xffffff00) | (wx >> 24);
+              break;
+    case 30:  w1[3] = (w1[3] & 0x0000ffff) | (wx << 16);
+              w2[0] = (w2[0] & 0xffff0000) | (wx >> 16);
+              break;
+    case 31:  w1[3] = (w1[3] & 0x00ffffff) | (wx << 24);
+              w2[0] = (w2[0] & 0xff000000) | (wx >>  8);
+              break;
+    case 32:  w2[0] =  wx;
+              break;
+    case 33:  w2[0] = (w2[0] & 0x000000ff) | (wx <<  8);
+              w2[1] = (w2[1] & 0xffffff00) | (wx >> 24);
+              break;
+    case 34:  w2[0] = (w2[0] & 0x0000ffff) | (wx << 16);
+              w2[1] = (w2[1] & 0xffff0000) | (wx >> 16);
+              break;
+    case 35:  w2[0] = (w2[0] & 0x00ffffff) | (wx << 24);
+              w2[1] = (w2[1] & 0xff000000) | (wx >>  8);
+              break;
+    case 36:  w2[1] =  wx;
+              break;
+    case 37:  w2[1] = (w2[1] & 0x000000ff) | (wx <<  8);
+              w2[2] = (w2[2] & 0xffffff00) | (wx >> 24);
+              break;
+    case 38:  w2[1] = (w2[1] & 0x0000ffff) | (wx << 16);
+              w2[2] = (w2[2] & 0xffff0000) | (wx >> 16);
+              break;
+    case 39:  w2[1] = (w2[1] & 0x00ffffff) | (wx << 24);
+              w2[2] = (w2[2] & 0xff000000) | (wx >>  8);
+              break;
+    case 40:  w2[2] =  wx;
+              break;
+    case 41:  w2[2] = (w2[2] & 0x000000ff) | (wx <<  8);
+              w2[3] = (w2[3] & 0xffffff00) | (wx >> 24);
+              break;
+    case 42:  w2[2] = (w2[2] & 0x0000ffff) | (wx << 16);
+              w2[3] = (w2[3] & 0xffff0000) | (wx >> 16);
+              break;
+    case 43:  w2[2] = (w2[2] & 0x00ffffff) | (wx << 24);
+              w2[3] = (w2[3] & 0xff000000) | (wx >>  8);
+              break;
+    case 44:  w2[3] =  wx;
+              break;
+    case 45:  w2[3] = (w2[3] & 0x000000ff) | (wx <<  8);
+              w3[0] = (w3[0] & 0xffffff00) | (wx >> 24);
+              break;
+    case 46:  w2[3] = (w2[3] & 0x0000ffff) | (wx << 16);
+              w3[0] = (w3[0] & 0xffff0000) | (wx >> 16);
+              break;
+    case 47:  w2[3] = (w2[3] & 0x00ffffff) | (wx << 24);
+              w3[0] = (w3[0] & 0xff000000) | (wx >>  8);
+              break;
+    case 48:  w3[0] =  wx;
+              break;
+    case 49:  w3[0] = (w3[0] & 0x000000ff) | (wx <<  8);
+              w3[1] = (w3[1] & 0xffffff00) | (wx >> 24);
+              break;
+    case 50:  w3[0] = (w3[0] & 0x0000ffff) | (wx << 16);
+              w3[1] = (w3[1] & 0xffff0000) | (wx >> 16);
+              break;
+    case 51:  w3[0] = (w3[0] & 0x00ffffff) | (wx << 24);
+              w3[1] = (w3[1] & 0xff000000) | (wx >>  8);
+              break;
+    case 52:  w3[1] =  wx;
+              break;
+    case 53:  w3[1] = (w3[1] & 0x000000ff) | (wx <<  8);
+              w3[2] = (w3[2] & 0xffffff00) | (wx >> 24);
+              break;
+    case 54:  w3[1] = (w3[1] & 0x0000ffff) | (wx << 16);
+              w3[2] = (w3[2] & 0xffff0000) | (wx >> 16);
+              break;
+    case 55:  w3[1] = (w3[1] & 0x00ffffff) | (wx << 24);
+              w3[2] = (w3[2] & 0xff000000) | (wx >>  8);
+              break;
+    case 56:  w3[2] =  wx;
+              break;
+    case 57:  w3[2] = (w3[2] & 0x000000ff) | (wx <<  8);
+              w3[3] = (w3[3] & 0xffffff00) | (wx >> 24);
+              break;
+    case 58:  w3[2] = (w3[2] & 0x0000ffff) | (wx << 16);
+              w3[3] = (w3[3] & 0xffff0000) | (wx >> 16);
+              break;
+    case 59:  w3[2] = (w3[2] & 0x00ffffff) | (wx << 24);
+              w3[3] = (w3[3] & 0xff000000) | (wx >>  8);
+              break;
+    case 60:  w3[3] =  wx;
+              break;
+    case 61:  w3[3] = (w3[3] & 0x000000ff) | (wx <<  8);
+              //w4[0] = (w4[0] & 0xffffff00) | (wx >> 24);
+              break;
+    case 62:  w3[3] = (w3[3] & 0x0000ffff) | (wx << 16);
+              //w4[0] = (w4[0] & 0xffff0000) | (wx >> 16);
+              break;
+    case 63:  w3[3] = (w3[3] & 0x00ffffff) | (wx << 24);
+              //w4[0] = (w4[0] & 0xff000000) | (wx >>  8);
+              break;
+  }
+  #endif
+}
+
+inline void overwrite_at_be_4x4 (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x wx, const u32 salt_len)
+{
+  // would be nice to have optimization based on amd_bytealign as with _le counterpart
+
+  switch (salt_len)
+  {
+    case  0:  w0[0] =  wx;
+              break;
+    case  1:  w0[0] = (w0[0] & 0xff000000) | (wx >>  8);
+              w0[1] = (w0[1] & 0x00ffffff) | (wx << 24);
+              break;
+    case  2:  w0[0] = (w0[0] & 0xffff0000) | (wx >> 16);
+              w0[1] = (w0[1] & 0x0000ffff) | (wx << 16);
+              break;
+    case  3:  w0[0] = (w0[0] & 0xffffff00) | (wx >> 24);
+              w0[1] = (w0[1] & 0x000000ff) | (wx <<  8);
+              break;
+    case  4:  w0[1] =  wx;
+              break;
+    case  5:  w0[1] = (w0[1] & 0xff000000) | (wx >>  8);
+              w0[2] = (w0[2] & 0x00ffffff) | (wx << 24);
+              break;
+    case  6:  w0[1] = (w0[1] & 0xffff0000) | (wx >> 16);
+              w0[2] = (w0[2] & 0x0000ffff) | (wx << 16);
+              break;
+    case  7:  w0[1] = (w0[1] & 0xffffff00) | (wx >> 24);
+              w0[2] = (w0[2] & 0x000000ff) | (wx <<  8);
+              break;
+    case  8:  w0[2] =  wx;
+              break;
+    case  9:  w0[2] = (w0[2] & 0xff000000) | (wx >>  8);
+              w0[3] = (w0[3] & 0x00ffffff) | (wx << 24);
+              break;
+    case 10:  w0[2] = (w0[2] & 0xffff0000) | (wx >> 16);
+              w0[3] = (w0[3] & 0x0000ffff) | (wx << 16);
+              break;
+    case 11:  w0[2] = (w0[2] & 0xffffff00) | (wx >> 24);
+              w0[3] = (w0[3] & 0x000000ff) | (wx <<  8);
+              break;
+    case 12:  w0[3] =  wx;
+              break;
+    case 13:  w0[3] = (w0[3] & 0xff000000) | (wx >>  8);
+              w1[0] = (w1[0] & 0x00ffffff) | (wx << 24);
+              break;
+    case 14:  w0[3] = (w0[3] & 0xffff0000) | (wx >> 16);
+              w1[0] = (w1[0] & 0x0000ffff) | (wx << 16);
+              break;
+    case 15:  w0[3] = (w0[3] & 0xffffff00) | (wx >> 24);
+              w1[0] = (w1[0] & 0x000000ff) | (wx <<  8);
+              break;
+    case 16:  w1[0] =  wx;
+              break;
+    case 17:  w1[0] = (w1[0] & 0xff000000) | (wx >>  8);
+              w1[1] = (w1[1] & 0x00ffffff) | (wx << 24);
+              break;
+    case 18:  w1[0] = (w1[0] & 0xffff0000) | (wx >> 16);
+              w1[1] = (w1[1] & 0x0000ffff) | (wx << 16);
+              break;
+    case 19:  w1[0] = (w1[0] & 0xffffff00) | (wx >> 24);
+              w1[1] = (w1[1] & 0x000000ff) | (wx <<  8);
+              break;
+    case 20:  w1[1] =  wx;
+              break;
+    case 21:  w1[1] = (w1[1] & 0xff000000) | (wx >>  8);
+              w1[2] = (w1[2] & 0x00ffffff) | (wx << 24);
+              break;
+    case 22:  w1[1] = (w1[1] & 0xffff0000) | (wx >> 16);
+              w1[2] = (w1[2] & 0x0000ffff) | (wx << 16);
+              break;
+    case 23:  w1[1] = (w1[1] & 0xffffff00) | (wx >> 24);
+              w1[2] = (w1[2] & 0x000000ff) | (wx <<  8);
+              break;
+    case 24:  w1[2] =  wx;
+              break;
+    case 25:  w1[2] = (w1[2] & 0xff000000) | (wx >>  8);
+              w1[3] = (w1[3] & 0x00ffffff) | (wx << 24);
+              break;
+    case 26:  w1[2] = (w1[2] & 0xffff0000) | (wx >> 16);
+              w1[3] = (w1[3] & 0x0000ffff) | (wx << 16);
+              break;
+    case 27:  w1[2] = (w1[2] & 0xffffff00) | (wx >> 24);
+              w1[3] = (w1[3] & 0x000000ff) | (wx <<  8);
+              break;
+    case 28:  w1[3] =  wx;
+              break;
+    case 29:  w1[3] = (w1[3] & 0xff000000) | (wx >>  8);
+              w2[0] = (w2[0] & 0x00ffffff) | (wx << 24);
+              break;
+    case 30:  w1[3] = (w1[3] & 0xffff0000) | (wx >> 16);
+              w2[0] = (w2[0] & 0x0000ffff) | (wx << 16);
+              break;
+    case 31:  w1[3] = (w1[3] & 0xffffff00) | (wx >> 24);
+              w2[0] = (w2[0] & 0x000000ff) | (wx <<  8);
+              break;
+    case 32:  w2[0] =  wx;
+              break;
+    case 33:  w2[0] = (w2[0] & 0xff000000) | (wx >>  8);
+              w2[1] = (w2[1] & 0x00ffffff) | (wx << 24);
+              break;
+    case 34:  w2[0] = (w2[0] & 0xffff0000) | (wx >> 16);
+              w2[1] = (w2[1] & 0x0000ffff) | (wx << 16);
+              break;
+    case 35:  w2[0] = (w2[0] & 0xffffff00) | (wx >> 24);
+              w2[1] = (w2[1] & 0x000000ff) | (wx <<  8);
+              break;
+    case 36:  w2[1] =  wx;
+              break;
+    case 37:  w2[1] = (w2[1] & 0xff000000) | (wx >>  8);
+              w2[2] = (w2[2] & 0x00ffffff) | (wx << 24);
+              break;
+    case 38:  w2[1] = (w2[1] & 0xffff0000) | (wx >> 16);
+              w2[2] = (w2[2] & 0x0000ffff) | (wx << 16);
+              break;
+    case 39:  w2[1] = (w2[1] & 0xffffff00) | (wx >> 24);
+              w2[2] = (w2[2] & 0x000000ff) | (wx <<  8);
+              break;
+    case 40:  w2[2] =  wx;
+              break;
+    case 41:  w2[2] = (w2[2] & 0xff000000) | (wx >>  8);
+              w2[3] = (w2[3] & 0x00ffffff) | (wx << 24);
+              break;
+    case 42:  w2[2] = (w2[2] & 0xffff0000) | (wx >> 16);
+              w2[3] = (w2[3] & 0x0000ffff) | (wx << 16);
+              break;
+    case 43:  w2[2] = (w2[2] & 0xffffff00) | (wx >> 24);
+              w2[3] = (w2[3] & 0x000000ff) | (wx <<  8);
+              break;
+    case 44:  w2[3] =  wx;
+              break;
+    case 45:  w2[3] = (w2[3] & 0xff000000) | (wx >>  8);
+              w3[0] = (w3[0] & 0x00ffffff) | (wx << 24);
+              break;
+    case 46:  w2[3] = (w2[3] & 0xffff0000) | (wx >> 16);
+              w3[0] = (w3[0] & 0x0000ffff) | (wx << 16);
+              break;
+    case 47:  w2[3] = (w2[3] & 0xffffff00) | (wx >> 24);
+              w3[0] = (w3[0] & 0x000000ff) | (wx <<  8);
+              break;
+    case 48:  w3[0] =  wx;
+              break;
+    case 49:  w3[0] = (w3[0] & 0xff000000) | (wx >>  8);
+              w3[1] = (w3[1] & 0x00ffffff) | (wx << 24);
+              break;
+    case 50:  w3[0] = (w3[0] & 0xffff0000) | (wx >> 16);
+              w3[1] = (w3[1] & 0x0000ffff) | (wx << 16);
+              break;
+    case 51:  w3[0] = (w3[0] & 0xffffff00) | (wx >> 24);
+              w3[1] = (w3[1] & 0x000000ff) | (wx <<  8);
+              break;
+    case 52:  w3[1] =  wx;
+              break;
+    case 53:  w3[1] = (w3[1] & 0xff000000) | (wx >>  8);
+              w3[2] = (w3[2] & 0x00ffffff) | (wx << 24);
+              break;
+    case 54:  w3[1] = (w3[1] & 0xffff0000) | (wx >> 16);
+              w3[2] = (w3[2] & 0x0000ffff) | (wx << 16);
+              break;
+    case 55:  w3[1] = (w3[1] & 0xffffff00) | (wx >> 24);
+              w3[2] = (w3[2] & 0x000000ff) | (wx <<  8);
+              break;
+    case 56:  w3[2] =  wx;
+              break;
+    case 57:  w3[2] = (w3[2] & 0xff000000) | (wx >>  8);
+              w3[3] = (w3[3] & 0x00ffffff) | (wx << 24);
+              break;
+    case 58:  w3[2] = (w3[2] & 0xffff0000) | (wx >> 16);
+              w3[3] = (w3[3] & 0x0000ffff) | (wx << 16);
+              break;
+    case 59:  w3[2] = (w3[2] & 0xffffff00) | (wx >> 24);
+              w3[3] = (w3[3] & 0x000000ff) | (wx <<  8);
+              break;
+    case 60:  w3[3] =  wx;
+              break;
+    case 61:  w3[3] = (w3[3] & 0xff000000) | (wx >>  8);
+              //w4[0] = (w4[0] & 0x00ffffff) | (wx << 24);
+              break;
+    case 62:  w3[3] = (w3[3] & 0xffff0000) | (wx >> 16);
+              //w4[0] = (w4[0] & 0x0000ffff) | (wx << 16);
+              break;
+    case 63:  w3[3] = (w3[3] & 0xffffff00) | (wx >> 24);
+              //w4[0] = (w4[0] & 0x000000ff) | (wx <<  8);
+              break;
+  }
+}
+
+/**
+ * vector functions as scalar (for outer loop usage)
+ */
+
+inline void append_0x01_1x4_S (u32 w0[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0]  = 0x01;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0100;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x010000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x01000000;
+      break;
+
+    case 4:
+      w0[1] = 0x01;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0100;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x010000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x01000000;
+      break;
+
+    case 8:
+      w0[2] = 0x01;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0100;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x010000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x01000000;
+      break;
+
+    case 12:
+      w0[3] = 0x01;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0100;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x010000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x01000000;
+      break;
+  }
+}
+
+inline void append_0x01_2x4_S (u32 w0[4], u32 w1[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x01;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0100;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x010000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x01000000;
+      break;
+
+    case 4:
+      w0[1] = 0x01;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0100;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x010000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x01000000;
+      break;
+
+    case 8:
+      w0[2] = 0x01;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0100;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x010000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x01000000;
+      break;
+
+    case 12:
+      w0[3] = 0x01;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0100;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x010000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x01000000;
+      break;
+
+    case 16:
+      w1[0] = 0x01;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0100;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x010000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x01000000;
+      break;
+
+    case 20:
+      w1[1] = 0x01;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0100;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x010000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x01000000;
+      break;
+
+    case 24:
+      w1[2] = 0x01;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0100;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x010000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x01000000;
+      break;
+
+    case 28:
+      w1[3] = 0x01;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0100;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x010000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x01000000;
+      break;
+  }
+}
+
+inline void append_0x01_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x01;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0100;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x010000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x01000000;
+      break;
+
+    case 4:
+      w0[1] = 0x01;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0100;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x010000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x01000000;
+      break;
+
+    case 8:
+      w0[2] = 0x01;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0100;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x010000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x01000000;
+      break;
+
+    case 12:
+      w0[3] = 0x01;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0100;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x010000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x01000000;
+      break;
+
+    case 16:
+      w1[0] = 0x01;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0100;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x010000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x01000000;
+      break;
+
+    case 20:
+      w1[1] = 0x01;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0100;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x010000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x01000000;
+      break;
+
+    case 24:
+      w1[2] = 0x01;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0100;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x010000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x01000000;
+      break;
+
+    case 28:
+      w1[3] = 0x01;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0100;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x010000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x01000000;
+      break;
+
+    case 32:
+      w2[0] = 0x01;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x0100;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x010000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x01000000;
+      break;
+
+    case 36:
+      w2[1] = 0x01;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x0100;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x010000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x01000000;
+      break;
+
+    case 40:
+      w2[2] = 0x01;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x0100;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x010000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x01000000;
+      break;
+
+    case 44:
+      w2[3] = 0x01;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x0100;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x010000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x01000000;
+      break;
+  }
+}
+
+inline void append_0x01_4x4_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x01;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0100;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x010000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x01000000;
+      break;
+
+    case 4:
+      w0[1] = 0x01;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0100;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x010000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x01000000;
+      break;
+
+    case 8:
+      w0[2] = 0x01;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0100;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x010000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x01000000;
+      break;
+
+    case 12:
+      w0[3] = 0x01;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0100;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x010000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x01000000;
+      break;
+
+    case 16:
+      w1[0] = 0x01;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0100;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x010000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x01000000;
+      break;
+
+    case 20:
+      w1[1] = 0x01;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0100;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x010000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x01000000;
+      break;
+
+    case 24:
+      w1[2] = 0x01;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0100;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x010000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x01000000;
+      break;
+
+    case 28:
+      w1[3] = 0x01;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0100;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x010000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x01000000;
+      break;
+
+    case 32:
+      w2[0] = 0x01;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x0100;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x010000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x01000000;
+      break;
+
+    case 36:
+      w2[1] = 0x01;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x0100;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x010000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x01000000;
+      break;
+
+    case 40:
+      w2[2] = 0x01;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x0100;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x010000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x01000000;
+      break;
+
+    case 44:
+      w2[3] = 0x01;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x0100;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x010000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x01000000;
+      break;
+
+    case 48:
+      w3[0] = 0x01;
+      break;
+
+    case 49:
+      w3[0] = w3[0] | 0x0100;
+      break;
+
+    case 50:
+      w3[0] = w3[0] | 0x010000;
+      break;
+
+    case 51:
+      w3[0] = w3[0] | 0x01000000;
+      break;
+
+    case 52:
+      w3[1] = 0x01;
+      break;
+
+    case 53:
+      w3[1] = w3[1] | 0x0100;
+      break;
+
+    case 54:
+      w3[1] = w3[1] | 0x010000;
+      break;
+
+    case 55:
+      w3[1] = w3[1] | 0x01000000;
+      break;
+
+    case 56:
+      w3[2] = 0x01;
+      break;
+
+    case 57:
+      w3[2] = w3[2] | 0x0100;
+      break;
+
+    case 58:
+      w3[2] = w3[2] | 0x010000;
+      break;
+
+    case 59:
+      w3[2] = w3[2] | 0x01000000;
+      break;
+
+    case 60:
+      w3[3] = 0x01;
+      break;
+
+    case 61:
+      w3[3] = w3[3] | 0x0100;
+      break;
+
+    case 62:
+      w3[3] = w3[3] | 0x010000;
+      break;
+
+    case 63:
+      w3[3] = w3[3] | 0x01000000;
+      break;
+  }
+}
+
+inline void append_0x02_2x4_S (u32 w0[4], u32 w1[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x02;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0200;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x020000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x02000000;
+      break;
+
+    case 4:
+      w0[1] = 0x02;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0200;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x020000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x02000000;
+      break;
+
+    case 8:
+      w0[2] = 0x02;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0200;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x020000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x02000000;
+      break;
+
+    case 12:
+      w0[3] = 0x02;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0200;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x020000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x02000000;
+      break;
+
+    case 16:
+      w1[0] = 0x02;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0200;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x020000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x02000000;
+      break;
+
+    case 20:
+      w1[1] = 0x02;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0200;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x020000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x02000000;
+      break;
+
+    case 24:
+      w1[2] = 0x02;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0200;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x020000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x02000000;
+      break;
+
+    case 28:
+      w1[3] = 0x02;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0200;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x020000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x02000000;
+      break;
+  }
+}
+
+inline void append_0x02_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x02;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x0200;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x020000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x02000000;
+      break;
+
+    case 4:
+      w0[1] = 0x02;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x0200;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x020000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x02000000;
+      break;
+
+    case 8:
+      w0[2] = 0x02;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x0200;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x020000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x02000000;
+      break;
+
+    case 12:
+      w0[3] = 0x02;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x0200;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x020000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x02000000;
+      break;
+
+    case 16:
+      w1[0] = 0x02;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x0200;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x020000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x02000000;
+      break;
+
+    case 20:
+      w1[1] = 0x02;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x0200;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x020000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x02000000;
+      break;
+
+    case 24:
+      w1[2] = 0x02;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x0200;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x020000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x02000000;
+      break;
+
+    case 28:
+      w1[3] = 0x02;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x0200;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x020000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x02000000;
+      break;
+
+    case 32:
+      w2[0] = 0x02;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x0200;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x020000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x02000000;
+      break;
+
+    case 36:
+      w2[1] = 0x02;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x0200;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x020000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x02000000;
+      break;
+
+    case 40:
+      w2[2] = 0x02;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x0200;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x020000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x02000000;
+      break;
+
+    case 44:
+      w2[3] = 0x02;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x0200;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x020000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x02000000;
+      break;
+  }
+}
+
+inline void append_0x80_1x4_S (u32 w0[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0]  = 0x80;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x8000;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x800000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x80000000;
+      break;
+
+    case 4:
+      w0[1] = 0x80;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x8000;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x800000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x80000000;
+      break;
+
+    case 8:
+      w0[2] = 0x80;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x8000;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x800000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x80000000;
+      break;
+
+    case 12:
+      w0[3] = 0x80;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x8000;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x800000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x80000000;
+      break;
+  }
+}
+
+inline void append_0x80_2x4_S (u32 w0[4], u32 w1[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x80;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x8000;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x800000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x80000000;
+      break;
+
+    case 4:
+      w0[1] = 0x80;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x8000;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x800000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x80000000;
+      break;
+
+    case 8:
+      w0[2] = 0x80;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x8000;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x800000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x80000000;
+      break;
+
+    case 12:
+      w0[3] = 0x80;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x8000;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x800000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x80000000;
+      break;
+
+    case 16:
+      w1[0] = 0x80;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x8000;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x800000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x80000000;
+      break;
+
+    case 20:
+      w1[1] = 0x80;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x8000;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x800000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x80000000;
+      break;
+
+    case 24:
+      w1[2] = 0x80;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x8000;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x800000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x80000000;
+      break;
+
+    case 28:
+      w1[3] = 0x80;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x8000;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x800000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x80000000;
+      break;
+  }
+}
+
+inline void append_0x80_3x4_S (u32 w0[4], u32 w1[4], u32 w2[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x80;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x8000;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x800000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x80000000;
+      break;
+
+    case 4:
+      w0[1] = 0x80;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x8000;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x800000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x80000000;
+      break;
+
+    case 8:
+      w0[2] = 0x80;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x8000;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x800000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x80000000;
+      break;
+
+    case 12:
+      w0[3] = 0x80;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x8000;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x800000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x80000000;
+      break;
+
+    case 16:
+      w1[0] = 0x80;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x8000;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x800000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x80000000;
+      break;
+
+    case 20:
+      w1[1] = 0x80;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x8000;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x800000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x80000000;
+      break;
+
+    case 24:
+      w1[2] = 0x80;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x8000;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x800000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x80000000;
+      break;
+
+    case 28:
+      w1[3] = 0x80;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x8000;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x800000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x80000000;
+      break;
+
+    case 32:
+      w2[0] = 0x80;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x8000;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x800000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x80000000;
+      break;
+
+    case 36:
+      w2[1] = 0x80;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x8000;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x800000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x80000000;
+      break;
+
+    case 40:
+      w2[2] = 0x80;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x8000;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x800000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x80000000;
+      break;
+
+    case 44:
+      w2[3] = 0x80;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x8000;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x800000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x80000000;
+      break;
+  }
+}
+
+inline void append_0x80_4x4_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset)
+{
+  switch (offset)
+  {
+    case 0:
+      w0[0] = 0x80;
+      break;
+
+    case 1:
+      w0[0] = w0[0] | 0x8000;
+      break;
+
+    case 2:
+      w0[0] = w0[0] | 0x800000;
+      break;
+
+    case 3:
+      w0[0] = w0[0] | 0x80000000;
+      break;
+
+    case 4:
+      w0[1] = 0x80;
+      break;
+
+    case 5:
+      w0[1] = w0[1] | 0x8000;
+      break;
+
+    case 6:
+      w0[1] = w0[1] | 0x800000;
+      break;
+
+    case 7:
+      w0[1] = w0[1] | 0x80000000;
+      break;
+
+    case 8:
+      w0[2] = 0x80;
+      break;
+
+    case 9:
+      w0[2] = w0[2] | 0x8000;
+      break;
+
+    case 10:
+      w0[2] = w0[2] | 0x800000;
+      break;
+
+    case 11:
+      w0[2] = w0[2] | 0x80000000;
+      break;
+
+    case 12:
+      w0[3] = 0x80;
+      break;
+
+    case 13:
+      w0[3] = w0[3] | 0x8000;
+      break;
+
+    case 14:
+      w0[3] = w0[3] | 0x800000;
+      break;
+
+    case 15:
+      w0[3] = w0[3] | 0x80000000;
+      break;
+
+    case 16:
+      w1[0] = 0x80;
+      break;
+
+    case 17:
+      w1[0] = w1[0] | 0x8000;
+      break;
+
+    case 18:
+      w1[0] = w1[0] | 0x800000;
+      break;
+
+    case 19:
+      w1[0] = w1[0] | 0x80000000;
+      break;
+
+    case 20:
+      w1[1] = 0x80;
+      break;
+
+    case 21:
+      w1[1] = w1[1] | 0x8000;
+      break;
+
+    case 22:
+      w1[1] = w1[1] | 0x800000;
+      break;
+
+    case 23:
+      w1[1] = w1[1] | 0x80000000;
+      break;
+
+    case 24:
+      w1[2] = 0x80;
+      break;
+
+    case 25:
+      w1[2] = w1[2] | 0x8000;
+      break;
+
+    case 26:
+      w1[2] = w1[2] | 0x800000;
+      break;
+
+    case 27:
+      w1[2] = w1[2] | 0x80000000;
+      break;
+
+    case 28:
+      w1[3] = 0x80;
+      break;
+
+    case 29:
+      w1[3] = w1[3] | 0x8000;
+      break;
+
+    case 30:
+      w1[3] = w1[3] | 0x800000;
+      break;
+
+    case 31:
+      w1[3] = w1[3] | 0x80000000;
+      break;
+
+    case 32:
+      w2[0] = 0x80;
+      break;
+
+    case 33:
+      w2[0] = w2[0] | 0x8000;
+      break;
+
+    case 34:
+      w2[0] = w2[0] | 0x800000;
+      break;
+
+    case 35:
+      w2[0] = w2[0] | 0x80000000;
+      break;
+
+    case 36:
+      w2[1] = 0x80;
+      break;
+
+    case 37:
+      w2[1] = w2[1] | 0x8000;
+      break;
+
+    case 38:
+      w2[1] = w2[1] | 0x800000;
+      break;
+
+    case 39:
+      w2[1] = w2[1] | 0x80000000;
+      break;
+
+    case 40:
+      w2[2] = 0x80;
+      break;
+
+    case 41:
+      w2[2] = w2[2] | 0x8000;
+      break;
+
+    case 42:
+      w2[2] = w2[2] | 0x800000;
+      break;
+
+    case 43:
+      w2[2] = w2[2] | 0x80000000;
+      break;
+
+    case 44:
+      w2[3] = 0x80;
+      break;
+
+    case 45:
+      w2[3] = w2[3] | 0x8000;
+      break;
+
+    case 46:
+      w2[3] = w2[3] | 0x800000;
+      break;
+
+    case 47:
+      w2[3] = w2[3] | 0x80000000;
+      break;
+
+    case 48:
+      w3[0] = 0x80;
+      break;
+
+    case 49:
+      w3[0] = w3[0] | 0x8000;
+      break;
+
+    case 50:
+      w3[0] = w3[0] | 0x800000;
+      break;
+
+    case 51:
+      w3[0] = w3[0] | 0x80000000;
+      break;
+
+    case 52:
+      w3[1] = 0x80;
+      break;
+
+    case 53:
+      w3[1] = w3[1] | 0x8000;
+      break;
+
+    case 54:
+      w3[1] = w3[1] | 0x800000;
+      break;
+
+    case 55:
+      w3[1] = w3[1] | 0x80000000;
+      break;
+
+    case 56:
+      w3[2] = 0x80;
+      break;
+
+    case 57:
+      w3[2] = w3[2] | 0x8000;
+      break;
+
+    case 58:
+      w3[2] = w3[2] | 0x800000;
+      break;
+
+    case 59:
+      w3[2] = w3[2] | 0x80000000;
+      break;
+
+    case 60:
+      w3[3] = 0x80;
+      break;
+
+    case 61:
+      w3[3] = w3[3] | 0x8000;
+      break;
+
+    case 62:
+      w3[3] = w3[3] | 0x800000;
+      break;
+
+    case 63:
+      w3[3] = w3[3] | 0x80000000;
+      break;
+  }
+}
+
+inline void truncate_block_S (u32 w[4], const u32 len)
+{
+  switch (len)
+  {
+    case  0:  w[0] &= 0;
+              w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  1:  w[0] &= 0x000000FF;
+              w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  2:  w[0] &= 0x0000FFFF;
+              w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  3:  w[0] &= 0x00FFFFFF;
+              w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  4:  w[1] &= 0;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  5:  w[1] &= 0x000000FF;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  6:  w[1] &= 0x0000FFFF;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  7:  w[1] &= 0x00FFFFFF;
+              w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  8:  w[2] &= 0;
+              w[3] &= 0;
+              break;
+    case  9:  w[2] &= 0x000000FF;
+              w[3] &= 0;
+              break;
+    case 10:  w[2] &= 0x0000FFFF;
+              w[3] &= 0;
+              break;
+    case 11:  w[2] &= 0x00FFFFFF;
+              w[3] &= 0;
+              break;
+    case 12:  w[3] &= 0;
+              break;
+    case 13:  w[3] &= 0x000000FF;
+              break;
+    case 14:  w[3] &= 0x0000FFFF;
+              break;
+    case 15:  w[3] &= 0x00FFFFFF;
+              break;
+  }
+}
+
+inline void make_unicode_S (const u32 in[4], u32 out1[4], u32 out2[4])
+{
+  #ifdef IS_NV
+  out2[3] = __byte_perm_S (in[3], 0, 0x7372);
+  out2[2] = __byte_perm_S (in[3], 0, 0x7170);
+  out2[1] = __byte_perm_S (in[2], 0, 0x7372);
+  out2[0] = __byte_perm_S (in[2], 0, 0x7170);
+  out1[3] = __byte_perm_S (in[1], 0, 0x7372);
+  out1[2] = __byte_perm_S (in[1], 0, 0x7170);
+  out1[1] = __byte_perm_S (in[0], 0, 0x7372);
+  out1[0] = __byte_perm_S (in[0], 0, 0x7170);
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  out2[3]  = ((in[3] >> 8) & 0x00FF0000) | ((in[3] >> 16) & 0x000000FF);
+  out2[2]  = ((in[3] << 8) & 0x00FF0000) | ((in[3] >>  0) & 0x000000FF);
+  out2[1]  = ((in[2] >> 8) & 0x00FF0000) | ((in[2] >> 16) & 0x000000FF);
+  out2[0]  = ((in[2] << 8) & 0x00FF0000) | ((in[2] >>  0) & 0x000000FF);
+  out1[3]  = ((in[1] >> 8) & 0x00FF0000) | ((in[1] >> 16) & 0x000000FF);
+  out1[2]  = ((in[1] << 8) & 0x00FF0000) | ((in[1] >>  0) & 0x000000FF);
+  out1[1]  = ((in[0] >> 8) & 0x00FF0000) | ((in[0] >> 16) & 0x000000FF);
+  out1[0]  = ((in[0] << 8) & 0x00FF0000) | ((in[0] >>  0) & 0x000000FF);
+  #endif
+}
+
+inline void undo_unicode_S (const u32 in1[4], const u32 in2[4], u32 out[4])
+{
+  #ifdef IS_NV
+  out[0] = __byte_perm_S (in1[0], in1[1], 0x6420);
+  out[1] = __byte_perm_S (in1[2], in1[3], 0x6420);
+  out[2] = __byte_perm_S (in2[0], in2[1], 0x6420);
+  out[3] = __byte_perm_S (in2[2], in2[3], 0x6420);
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  out[0] = ((in1[0] & 0x000000ff) >>  0) | ((in1[0] & 0x00ff0000) >>  8)
+         | ((in1[1] & 0x000000ff) << 16) | ((in1[1] & 0x00ff0000) <<  8);
+  out[1] = ((in1[2] & 0x000000ff) >>  0) | ((in1[2] & 0x00ff0000) >>  8)
+         | ((in1[3] & 0x000000ff) << 16) | ((in1[3] & 0x00ff0000) <<  8);
+  out[2] = ((in2[0] & 0x000000ff) >>  0) | ((in2[0] & 0x00ff0000) >>  8)
+         | ((in2[1] & 0x000000ff) << 16) | ((in2[1] & 0x00ff0000) <<  8);
+  out[3] = ((in2[2] & 0x000000ff) >>  0) | ((in2[2] & 0x00ff0000) >>  8)
+         | ((in2[3] & 0x000000ff) << 16) | ((in2[3] & 0x00ff0000) <<  8);
+  #endif
+}
+
+inline void switch_buffer_by_offset_le_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset)
+{
+  #if defined IS_AMD || defined IS_GENERIC
+  const int offset_mod_4 = offset & 3;
+
+  const int offset_minus_4 = 4 - offset;
+
+  switch (offset / 4)
+  {
+    case 0:
+      w3[2] = amd_bytealign_S (    0, w3[1], offset_minus_4);
+      w3[1] = amd_bytealign_S (w3[1], w3[0], offset_minus_4);
+      w3[0] = amd_bytealign_S (w3[0], w2[3], offset_minus_4);
+      w2[3] = amd_bytealign_S (w2[3], w2[2], offset_minus_4);
+      w2[2] = amd_bytealign_S (w2[2], w2[1], offset_minus_4);
+      w2[1] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
+      w2[0] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
+      w1[3] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
+      w1[2] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
+      w1[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
+      w1[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w0[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w0[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w0[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w0[0] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+
+      if (offset_mod_4 == 0)
+      {
+        w0[0] = w0[1];
+        w0[1] = w0[2];
+        w0[2] = w0[3];
+        w0[3] = w1[0];
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 1:
+      w3[2] = amd_bytealign_S (    0, w3[0], offset_minus_4);
+      w3[1] = amd_bytealign_S (w3[0], w2[3], offset_minus_4);
+      w3[0] = amd_bytealign_S (w2[3], w2[2], offset_minus_4);
+      w2[3] = amd_bytealign_S (w2[2], w2[1], offset_minus_4);
+      w2[2] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
+      w2[1] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
+      w2[0] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
+      w1[3] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
+      w1[2] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
+      w1[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w1[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w0[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w0[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w0[1] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w0[1] = w0[2];
+        w0[2] = w0[3];
+        w0[3] = w1[0];
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 2:
+      w3[2] = amd_bytealign_S (    0, w2[3], offset_minus_4);
+      w3[1] = amd_bytealign_S (w2[3], w2[2], offset_minus_4);
+      w3[0] = amd_bytealign_S (w2[2], w2[1], offset_minus_4);
+      w2[3] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
+      w2[2] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
+      w2[1] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
+      w2[0] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
+      w1[3] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
+      w1[2] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w1[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w1[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w0[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w0[2] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w0[2] = w0[3];
+        w0[3] = w1[0];
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 3:
+      w3[2] = amd_bytealign_S (    0, w2[2], offset_minus_4);
+      w3[1] = amd_bytealign_S (w2[2], w2[1], offset_minus_4);
+      w3[0] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
+      w2[3] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
+      w2[2] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
+      w2[1] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
+      w2[0] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
+      w1[3] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w1[2] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w1[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w1[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w0[3] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w0[3] = w1[0];
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 4:
+      w3[2] = amd_bytealign_S (    0, w2[1], offset_minus_4);
+      w3[1] = amd_bytealign_S (w2[1], w2[0], offset_minus_4);
+      w3[0] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
+      w2[3] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
+      w2[2] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
+      w2[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
+      w2[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w1[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w1[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w1[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w1[0] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w1[0] = w1[1];
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 5:
+      w3[2] = amd_bytealign_S (    0, w2[0], offset_minus_4);
+      w3[1] = amd_bytealign_S (w2[0], w1[3], offset_minus_4);
+      w3[0] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
+      w2[3] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
+      w2[2] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
+      w2[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w2[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w1[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w1[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w1[1] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w1[1] = w1[2];
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 6:
+      w3[2] = amd_bytealign_S (    0, w1[3], offset_minus_4);
+      w3[1] = amd_bytealign_S (w1[3], w1[2], offset_minus_4);
+      w3[0] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
+      w2[3] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
+      w2[2] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w2[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w2[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w1[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w1[2] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w1[2] = w1[3];
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 7:
+      w3[2] = amd_bytealign_S (    0, w1[2], offset_minus_4);
+      w3[1] = amd_bytealign_S (w1[2], w1[1], offset_minus_4);
+      w3[0] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
+      w2[3] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w2[2] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w2[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w2[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w1[3] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w1[3] = w2[0];
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 8:
+      w3[2] = amd_bytealign_S (    0, w1[1], offset_minus_4);
+      w3[1] = amd_bytealign_S (w1[1], w1[0], offset_minus_4);
+      w3[0] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w2[3] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w2[2] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w2[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w2[0] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w2[0] = w2[1];
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 9:
+      w3[2] = amd_bytealign_S (    0, w1[0], offset_minus_4);
+      w3[1] = amd_bytealign_S (w1[0], w0[3], offset_minus_4);
+      w3[0] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w2[3] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w2[2] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w2[1] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w2[1] = w2[2];
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 10:
+      w3[2] = amd_bytealign_S (    0, w0[3], offset_minus_4);
+      w3[1] = amd_bytealign_S (w0[3], w0[2], offset_minus_4);
+      w3[0] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w2[3] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w2[2] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w2[2] = w2[3];
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 11:
+      w3[2] = amd_bytealign_S (    0, w0[2], offset_minus_4);
+      w3[1] = amd_bytealign_S (w0[2], w0[1], offset_minus_4);
+      w3[0] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w2[3] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w2[3] = w3[0];
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 12:
+      w3[2] = amd_bytealign_S (    0, w0[1], offset_minus_4);
+      w3[1] = amd_bytealign_S (w0[1], w0[0], offset_minus_4);
+      w3[0] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w3[0] = w3[1];
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+
+    case 13:
+      w3[2] = amd_bytealign_S (    0, w0[0], offset_minus_4);
+      w3[1] = amd_bytealign_S (w0[0],     0, offset_minus_4);
+      w3[0] = 0;
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      if (offset_mod_4 == 0)
+      {
+        w3[1] = w3[2];
+        w3[2] = 0;
+      }
+
+      break;
+  }
+  #endif
+
+  #ifdef IS_NV
+  const int offset_minus_4 = 4 - (offset % 4);
+
+  const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff;
+
+  switch (offset / 4)
+  {
+    case 0:
+      w3[1] = __byte_perm_S (w3[0], w3[1], selector);
+      w3[0] = __byte_perm_S (w2[3], w3[0], selector);
+      w2[3] = __byte_perm_S (w2[2], w2[3], selector);
+      w2[2] = __byte_perm_S (w2[1], w2[2], selector);
+      w2[1] = __byte_perm_S (w2[0], w2[1], selector);
+      w2[0] = __byte_perm_S (w1[3], w2[0], selector);
+      w1[3] = __byte_perm_S (w1[2], w1[3], selector);
+      w1[2] = __byte_perm_S (w1[1], w1[2], selector);
+      w1[1] = __byte_perm_S (w1[0], w1[1], selector);
+      w1[0] = __byte_perm_S (w0[3], w1[0], selector);
+      w0[3] = __byte_perm_S (w0[2], w0[3], selector);
+      w0[2] = __byte_perm_S (w0[1], w0[2], selector);
+      w0[1] = __byte_perm_S (w0[0], w0[1], selector);
+      w0[0] = __byte_perm_S (    0, w0[0], selector);
+
+      break;
+
+    case 1:
+      w3[1] = __byte_perm_S (w2[3], w3[0], selector);
+      w3[0] = __byte_perm_S (w2[2], w2[3], selector);
+      w2[3] = __byte_perm_S (w2[1], w2[2], selector);
+      w2[2] = __byte_perm_S (w2[0], w2[1], selector);
+      w2[1] = __byte_perm_S (w1[3], w2[0], selector);
+      w2[0] = __byte_perm_S (w1[2], w1[3], selector);
+      w1[3] = __byte_perm_S (w1[1], w1[2], selector);
+      w1[2] = __byte_perm_S (w1[0], w1[1], selector);
+      w1[1] = __byte_perm_S (w0[3], w1[0], selector);
+      w1[0] = __byte_perm_S (w0[2], w0[3], selector);
+      w0[3] = __byte_perm_S (w0[1], w0[2], selector);
+      w0[2] = __byte_perm_S (w0[0], w0[1], selector);
+      w0[1] = __byte_perm_S (    0, w0[0], selector);
+      w0[0] = 0;
+
+      break;
+
+    case 2:
+      w3[1] = __byte_perm_S (w2[2], w2[3], selector);
+      w3[0] = __byte_perm_S (w2[1], w2[2], selector);
+      w2[3] = __byte_perm_S (w2[0], w2[1], selector);
+      w2[2] = __byte_perm_S (w1[3], w2[0], selector);
+      w2[1] = __byte_perm_S (w1[2], w1[3], selector);
+      w2[0] = __byte_perm_S (w1[1], w1[2], selector);
+      w1[3] = __byte_perm_S (w1[0], w1[1], selector);
+      w1[2] = __byte_perm_S (w0[3], w1[0], selector);
+      w1[1] = __byte_perm_S (w0[2], w0[3], selector);
+      w1[0] = __byte_perm_S (w0[1], w0[2], selector);
+      w0[3] = __byte_perm_S (w0[0], w0[1], selector);
+      w0[2] = __byte_perm_S (    0, w0[0], selector);
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 3:
+      w3[1] = __byte_perm_S (w2[1], w2[2], selector);
+      w3[0] = __byte_perm_S (w2[0], w2[1], selector);
+      w2[3] = __byte_perm_S (w1[3], w2[0], selector);
+      w2[2] = __byte_perm_S (w1[2], w1[3], selector);
+      w2[1] = __byte_perm_S (w1[1], w1[2], selector);
+      w2[0] = __byte_perm_S (w1[0], w1[1], selector);
+      w1[3] = __byte_perm_S (w0[3], w1[0], selector);
+      w1[2] = __byte_perm_S (w0[2], w0[3], selector);
+      w1[1] = __byte_perm_S (w0[1], w0[2], selector);
+      w1[0] = __byte_perm_S (w0[0], w0[1], selector);
+      w0[3] = __byte_perm_S (    0, w0[0], selector);
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 4:
+      w3[1] = __byte_perm_S (w2[0], w2[1], selector);
+      w3[0] = __byte_perm_S (w1[3], w2[0], selector);
+      w2[3] = __byte_perm_S (w1[2], w1[3], selector);
+      w2[2] = __byte_perm_S (w1[1], w1[2], selector);
+      w2[1] = __byte_perm_S (w1[0], w1[1], selector);
+      w2[0] = __byte_perm_S (w0[3], w1[0], selector);
+      w1[3] = __byte_perm_S (w0[2], w0[3], selector);
+      w1[2] = __byte_perm_S (w0[1], w0[2], selector);
+      w1[1] = __byte_perm_S (w0[0], w0[1], selector);
+      w1[0] = __byte_perm_S (    0, w0[0], selector);
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 5:
+      w3[1] = __byte_perm_S (w1[3], w2[0], selector);
+      w3[0] = __byte_perm_S (w1[2], w1[3], selector);
+      w2[3] = __byte_perm_S (w1[1], w1[2], selector);
+      w2[2] = __byte_perm_S (w1[0], w1[1], selector);
+      w2[1] = __byte_perm_S (w0[3], w1[0], selector);
+      w2[0] = __byte_perm_S (w0[2], w0[3], selector);
+      w1[3] = __byte_perm_S (w0[1], w0[2], selector);
+      w1[2] = __byte_perm_S (w0[0], w0[1], selector);
+      w1[1] = __byte_perm_S (    0, w0[0], selector);
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 6:
+      w3[1] = __byte_perm_S (w1[2], w1[3], selector);
+      w3[0] = __byte_perm_S (w1[1], w1[2], selector);
+      w2[3] = __byte_perm_S (w1[0], w1[1], selector);
+      w2[2] = __byte_perm_S (w0[3], w1[0], selector);
+      w2[1] = __byte_perm_S (w0[2], w0[3], selector);
+      w2[0] = __byte_perm_S (w0[1], w0[2], selector);
+      w1[3] = __byte_perm_S (w0[0], w0[1], selector);
+      w1[2] = __byte_perm_S (    0, w0[0], selector);
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 7:
+      w3[1] = __byte_perm_S (w1[1], w1[2], selector);
+      w3[0] = __byte_perm_S (w1[0], w1[1], selector);
+      w2[3] = __byte_perm_S (w0[3], w1[0], selector);
+      w2[2] = __byte_perm_S (w0[2], w0[3], selector);
+      w2[1] = __byte_perm_S (w0[1], w0[2], selector);
+      w2[0] = __byte_perm_S (w0[0], w0[1], selector);
+      w1[3] = __byte_perm_S (    0, w0[0], selector);
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 8:
+      w3[1] = __byte_perm_S (w1[0], w1[1], selector);
+      w3[0] = __byte_perm_S (w0[3], w1[0], selector);
+      w2[3] = __byte_perm_S (w0[2], w0[3], selector);
+      w2[2] = __byte_perm_S (w0[1], w0[2], selector);
+      w2[1] = __byte_perm_S (w0[0], w0[1], selector);
+      w2[0] = __byte_perm_S (    0, w0[0], selector);
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 9:
+      w3[1] = __byte_perm_S (w0[3], w1[0], selector);
+      w3[0] = __byte_perm_S (w0[2], w0[3], selector);
+      w2[3] = __byte_perm_S (w0[1], w0[2], selector);
+      w2[2] = __byte_perm_S (w0[0], w0[1], selector);
+      w2[1] = __byte_perm_S (    0, w0[0], selector);
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 10:
+      w3[1] = __byte_perm_S (w0[2], w0[3], selector);
+      w3[0] = __byte_perm_S (w0[1], w0[2], selector);
+      w2[3] = __byte_perm_S (w0[0], w0[1], selector);
+      w2[2] = __byte_perm_S (    0, w0[0], selector);
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 11:
+      w3[1] = __byte_perm_S (w0[1], w0[2], selector);
+      w3[0] = __byte_perm_S (w0[0], w0[1], selector);
+      w2[3] = __byte_perm_S (    0, w0[0], selector);
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 12:
+      w3[1] = __byte_perm_S (w0[0], w0[1], selector);
+      w3[0] = __byte_perm_S (    0, w0[0], selector);
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+
+    case 13:
+      w3[1] = __byte_perm_S (    0, w0[0], selector);
+      w3[0] = 0;
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+
+      break;
+  }
+  #endif
+}
+
+inline void switch_buffer_by_offset_be_S (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 offset)
+{
+  #if defined IS_AMD || defined IS_GENERIC
+  switch (offset / 4)
+  {
+    case 0:
+      w3[2] = amd_bytealign_S (w3[1],     0, offset);
+      w3[1] = amd_bytealign_S (w3[0], w3[1], offset);
+      w3[0] = amd_bytealign_S (w2[3], w3[0], offset);
+      w2[3] = amd_bytealign_S (w2[2], w2[3], offset);
+      w2[2] = amd_bytealign_S (w2[1], w2[2], offset);
+      w2[1] = amd_bytealign_S (w2[0], w2[1], offset);
+      w2[0] = amd_bytealign_S (w1[3], w2[0], offset);
+      w1[3] = amd_bytealign_S (w1[2], w1[3], offset);
+      w1[2] = amd_bytealign_S (w1[1], w1[2], offset);
+      w1[1] = amd_bytealign_S (w1[0], w1[1], offset);
+      w1[0] = amd_bytealign_S (w0[3], w1[0], offset);
+      w0[3] = amd_bytealign_S (w0[2], w0[3], offset);
+      w0[2] = amd_bytealign_S (w0[1], w0[2], offset);
+      w0[1] = amd_bytealign_S (w0[0], w0[1], offset);
+      w0[0] = amd_bytealign_S (    0, w0[0], offset);
+      break;
+
+    case 1:
+      w3[2] = amd_bytealign_S (w3[0],     0, offset);
+      w3[1] = amd_bytealign_S (w2[3], w3[0], offset);
+      w3[0] = amd_bytealign_S (w2[2], w2[3], offset);
+      w2[3] = amd_bytealign_S (w2[1], w2[2], offset);
+      w2[2] = amd_bytealign_S (w2[0], w2[1], offset);
+      w2[1] = amd_bytealign_S (w1[3], w2[0], offset);
+      w2[0] = amd_bytealign_S (w1[2], w1[3], offset);
+      w1[3] = amd_bytealign_S (w1[1], w1[2], offset);
+      w1[2] = amd_bytealign_S (w1[0], w1[1], offset);
+      w1[1] = amd_bytealign_S (w0[3], w1[0], offset);
+      w1[0] = amd_bytealign_S (w0[2], w0[3], offset);
+      w0[3] = amd_bytealign_S (w0[1], w0[2], offset);
+      w0[2] = amd_bytealign_S (w0[0], w0[1], offset);
+      w0[1] = amd_bytealign_S (    0, w0[0], offset);
+      w0[0] = 0;
+      break;
+
+    case 2:
+      w3[2] = amd_bytealign_S (w2[3],     0, offset);
+      w3[1] = amd_bytealign_S (w2[2], w2[3], offset);
+      w3[0] = amd_bytealign_S (w2[1], w2[2], offset);
+      w2[3] = amd_bytealign_S (w2[0], w2[1], offset);
+      w2[2] = amd_bytealign_S (w1[3], w2[0], offset);
+      w2[1] = amd_bytealign_S (w1[2], w1[3], offset);
+      w2[0] = amd_bytealign_S (w1[1], w1[2], offset);
+      w1[3] = amd_bytealign_S (w1[0], w1[1], offset);
+      w1[2] = amd_bytealign_S (w0[3], w1[0], offset);
+      w1[1] = amd_bytealign_S (w0[2], w0[3], offset);
+      w1[0] = amd_bytealign_S (w0[1], w0[2], offset);
+      w0[3] = amd_bytealign_S (w0[0], w0[1], offset);
+      w0[2] = amd_bytealign_S (    0, w0[0], offset);
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 3:
+      w3[2] = amd_bytealign_S (w2[2],     0, offset);
+      w3[1] = amd_bytealign_S (w2[1], w2[2], offset);
+      w3[0] = amd_bytealign_S (w2[0], w2[1], offset);
+      w2[3] = amd_bytealign_S (w1[3], w2[0], offset);
+      w2[2] = amd_bytealign_S (w1[2], w1[3], offset);
+      w2[1] = amd_bytealign_S (w1[1], w1[2], offset);
+      w2[0] = amd_bytealign_S (w1[0], w1[1], offset);
+      w1[3] = amd_bytealign_S (w0[3], w1[0], offset);
+      w1[2] = amd_bytealign_S (w0[2], w0[3], offset);
+      w1[1] = amd_bytealign_S (w0[1], w0[2], offset);
+      w1[0] = amd_bytealign_S (w0[0], w0[1], offset);
+      w0[3] = amd_bytealign_S (    0, w0[0], offset);
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 4:
+      w3[2] = amd_bytealign_S (w2[1],     0, offset);
+      w3[1] = amd_bytealign_S (w2[0], w2[1], offset);
+      w3[0] = amd_bytealign_S (w1[3], w2[0], offset);
+      w2[3] = amd_bytealign_S (w1[2], w1[3], offset);
+      w2[2] = amd_bytealign_S (w1[1], w1[2], offset);
+      w2[1] = amd_bytealign_S (w1[0], w1[1], offset);
+      w2[0] = amd_bytealign_S (w0[3], w1[0], offset);
+      w1[3] = amd_bytealign_S (w0[2], w0[3], offset);
+      w1[2] = amd_bytealign_S (w0[1], w0[2], offset);
+      w1[1] = amd_bytealign_S (w0[0], w0[1], offset);
+      w1[0] = amd_bytealign_S (    0, w0[0], offset);
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 5:
+      w3[2] = amd_bytealign_S (w2[0],     0, offset);
+      w3[1] = amd_bytealign_S (w1[3], w2[0], offset);
+      w3[0] = amd_bytealign_S (w1[2], w1[3], offset);
+      w2[3] = amd_bytealign_S (w1[1], w1[2], offset);
+      w2[2] = amd_bytealign_S (w1[0], w1[1], offset);
+      w2[1] = amd_bytealign_S (w0[3], w1[0], offset);
+      w2[0] = amd_bytealign_S (w0[2], w0[3], offset);
+      w1[3] = amd_bytealign_S (w0[1], w0[2], offset);
+      w1[2] = amd_bytealign_S (w0[0], w0[1], offset);
+      w1[1] = amd_bytealign_S (    0, w0[0], offset);
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 6:
+      w3[2] = amd_bytealign_S (w1[3],     0, offset);
+      w3[1] = amd_bytealign_S (w1[2], w1[3], offset);
+      w3[0] = amd_bytealign_S (w1[1], w1[2], offset);
+      w2[3] = amd_bytealign_S (w1[0], w1[1], offset);
+      w2[2] = amd_bytealign_S (w0[3], w1[0], offset);
+      w2[1] = amd_bytealign_S (w0[2], w0[3], offset);
+      w2[0] = amd_bytealign_S (w0[1], w0[2], offset);
+      w1[3] = amd_bytealign_S (w0[0], w0[1], offset);
+      w1[2] = amd_bytealign_S (    0, w0[0], offset);
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 7:
+      w3[2] = amd_bytealign_S (w1[2],     0, offset);
+      w3[1] = amd_bytealign_S (w1[1], w1[2], offset);
+      w3[0] = amd_bytealign_S (w1[0], w1[1], offset);
+      w2[3] = amd_bytealign_S (w0[3], w1[0], offset);
+      w2[2] = amd_bytealign_S (w0[2], w0[3], offset);
+      w2[1] = amd_bytealign_S (w0[1], w0[2], offset);
+      w2[0] = amd_bytealign_S (w0[0], w0[1], offset);
+      w1[3] = amd_bytealign_S (    0, w0[0], offset);
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 8:
+      w3[2] = amd_bytealign_S (w1[1],     0, offset);
+      w3[1] = amd_bytealign_S (w1[0], w1[1], offset);
+      w3[0] = amd_bytealign_S (w0[3], w1[0], offset);
+      w2[3] = amd_bytealign_S (w0[2], w0[3], offset);
+      w2[2] = amd_bytealign_S (w0[1], w0[2], offset);
+      w2[1] = amd_bytealign_S (w0[0], w0[1], offset);
+      w2[0] = amd_bytealign_S (    0, w0[0], offset);
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 9:
+      w3[2] = amd_bytealign_S (w1[0],     0, offset);
+      w3[1] = amd_bytealign_S (w0[3], w1[0], offset);
+      w3[0] = amd_bytealign_S (w0[2], w0[3], offset);
+      w2[3] = amd_bytealign_S (w0[1], w0[2], offset);
+      w2[2] = amd_bytealign_S (w0[0], w0[1], offset);
+      w2[1] = amd_bytealign_S (    0, w0[0], offset);
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 10:
+      w3[2] = amd_bytealign_S (w0[3],     0, offset);
+      w3[1] = amd_bytealign_S (w0[2], w0[3], offset);
+      w3[0] = amd_bytealign_S (w0[1], w0[2], offset);
+      w2[3] = amd_bytealign_S (w0[0], w0[1], offset);
+      w2[2] = amd_bytealign_S (    0, w0[0], offset);
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 11:
+      w3[2] = amd_bytealign_S (w0[2],     0, offset);
+      w3[1] = amd_bytealign_S (w0[1], w0[2], offset);
+      w3[0] = amd_bytealign_S (w0[0], w0[1], offset);
+      w2[3] = amd_bytealign_S (    0, w0[0], offset);
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 12:
+      w3[2] = amd_bytealign_S (w0[1],     0, offset);
+      w3[1] = amd_bytealign_S (w0[0], w0[1], offset);
+      w3[0] = amd_bytealign_S (    0, w0[0], offset);
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 13:
+      w3[2] = amd_bytealign_S (w0[0],     0, offset);
+      w3[1] = amd_bytealign_S (    0, w0[0], offset);
+      w3[0] = 0;
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+  }
+  #endif
+
+  #ifdef IS_NV
+  const int selector = (0x76543210 >> ((offset & 3) * 4)) & 0xffff;
+
+  switch (offset / 4)
+  {
+    case 0:
+      w3[1] = __byte_perm_S (w3[1], w3[0], selector);
+      w3[0] = __byte_perm_S (w3[0], w2[3], selector);
+      w2[3] = __byte_perm_S (w2[3], w2[2], selector);
+      w2[2] = __byte_perm_S (w2[2], w2[1], selector);
+      w2[1] = __byte_perm_S (w2[1], w2[0], selector);
+      w2[0] = __byte_perm_S (w2[0], w1[3], selector);
+      w1[3] = __byte_perm_S (w1[3], w1[2], selector);
+      w1[2] = __byte_perm_S (w1[2], w1[1], selector);
+      w1[1] = __byte_perm_S (w1[1], w1[0], selector);
+      w1[0] = __byte_perm_S (w1[0], w0[3], selector);
+      w0[3] = __byte_perm_S (w0[3], w0[2], selector);
+      w0[2] = __byte_perm_S (w0[2], w0[1], selector);
+      w0[1] = __byte_perm_S (w0[1], w0[0], selector);
+      w0[0] = __byte_perm_S (w0[0],     0, selector);
+      break;
+
+    case 1:
+      w3[1] = __byte_perm_S (w3[0], w2[3], selector);
+      w3[0] = __byte_perm_S (w2[3], w2[2], selector);
+      w2[3] = __byte_perm_S (w2[2], w2[1], selector);
+      w2[2] = __byte_perm_S (w2[1], w2[0], selector);
+      w2[1] = __byte_perm_S (w2[0], w1[3], selector);
+      w2[0] = __byte_perm_S (w1[3], w1[2], selector);
+      w1[3] = __byte_perm_S (w1[2], w1[1], selector);
+      w1[2] = __byte_perm_S (w1[1], w1[0], selector);
+      w1[1] = __byte_perm_S (w1[0], w0[3], selector);
+      w1[0] = __byte_perm_S (w0[3], w0[2], selector);
+      w0[3] = __byte_perm_S (w0[2], w0[1], selector);
+      w0[2] = __byte_perm_S (w0[1], w0[0], selector);
+      w0[1] = __byte_perm_S (w0[0],     0, selector);
+      w0[0] = 0;
+      break;
+
+    case 2:
+      w3[1] = __byte_perm_S (w2[3], w2[2], selector);
+      w3[0] = __byte_perm_S (w2[2], w2[1], selector);
+      w2[3] = __byte_perm_S (w2[1], w2[0], selector);
+      w2[2] = __byte_perm_S (w2[0], w1[3], selector);
+      w2[1] = __byte_perm_S (w1[3], w1[2], selector);
+      w2[0] = __byte_perm_S (w1[2], w1[1], selector);
+      w1[3] = __byte_perm_S (w1[1], w1[0], selector);
+      w1[2] = __byte_perm_S (w1[0], w0[3], selector);
+      w1[1] = __byte_perm_S (w0[3], w0[2], selector);
+      w1[0] = __byte_perm_S (w0[2], w0[1], selector);
+      w0[3] = __byte_perm_S (w0[1], w0[0], selector);
+      w0[2] = __byte_perm_S (w0[0],     0, selector);
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 3:
+      w3[1] = __byte_perm_S (w2[2], w2[1], selector);
+      w3[0] = __byte_perm_S (w2[1], w2[0], selector);
+      w2[3] = __byte_perm_S (w2[0], w1[3], selector);
+      w2[2] = __byte_perm_S (w1[3], w1[2], selector);
+      w2[1] = __byte_perm_S (w1[2], w1[1], selector);
+      w2[0] = __byte_perm_S (w1[1], w1[0], selector);
+      w1[3] = __byte_perm_S (w1[0], w0[3], selector);
+      w1[2] = __byte_perm_S (w0[3], w0[2], selector);
+      w1[1] = __byte_perm_S (w0[2], w0[1], selector);
+      w1[0] = __byte_perm_S (w0[1], w0[0], selector);
+      w0[3] = __byte_perm_S (w0[0],     0, selector);
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 4:
+      w3[1] = __byte_perm_S (w2[1], w2[0], selector);
+      w3[0] = __byte_perm_S (w2[0], w1[3], selector);
+      w2[3] = __byte_perm_S (w1[3], w1[2], selector);
+      w2[2] = __byte_perm_S (w1[2], w1[1], selector);
+      w2[1] = __byte_perm_S (w1[1], w1[0], selector);
+      w2[0] = __byte_perm_S (w1[0], w0[3], selector);
+      w1[3] = __byte_perm_S (w0[3], w0[2], selector);
+      w1[2] = __byte_perm_S (w0[2], w0[1], selector);
+      w1[1] = __byte_perm_S (w0[1], w0[0], selector);
+      w1[0] = __byte_perm_S (w0[0],     0, selector);
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 5:
+      w3[1] = __byte_perm_S (w2[0], w1[3], selector);
+      w3[0] = __byte_perm_S (w1[3], w1[2], selector);
+      w2[3] = __byte_perm_S (w1[2], w1[1], selector);
+      w2[2] = __byte_perm_S (w1[1], w1[0], selector);
+      w2[1] = __byte_perm_S (w1[0], w0[3], selector);
+      w2[0] = __byte_perm_S (w0[3], w0[2], selector);
+      w1[3] = __byte_perm_S (w0[2], w0[1], selector);
+      w1[2] = __byte_perm_S (w0[1], w0[0], selector);
+      w1[1] = __byte_perm_S (w0[0],     0, selector);
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 6:
+      w3[1] = __byte_perm_S (w1[3], w1[2], selector);
+      w3[0] = __byte_perm_S (w1[2], w1[1], selector);
+      w2[3] = __byte_perm_S (w1[1], w1[0], selector);
+      w2[2] = __byte_perm_S (w1[0], w0[3], selector);
+      w2[1] = __byte_perm_S (w0[3], w0[2], selector);
+      w2[0] = __byte_perm_S (w0[2], w0[1], selector);
+      w1[3] = __byte_perm_S (w0[1], w0[0], selector);
+      w1[2] = __byte_perm_S (w0[0],     0, selector);
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 7:
+      w3[1] = __byte_perm_S (w1[2], w1[1], selector);
+      w3[0] = __byte_perm_S (w1[1], w1[0], selector);
+      w2[3] = __byte_perm_S (w1[0], w0[3], selector);
+      w2[2] = __byte_perm_S (w0[3], w0[2], selector);
+      w2[1] = __byte_perm_S (w0[2], w0[1], selector);
+      w2[0] = __byte_perm_S (w0[1], w0[0], selector);
+      w1[3] = __byte_perm_S (w0[0],     0, selector);
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 8:
+      w3[1] = __byte_perm_S (w1[1], w1[0], selector);
+      w3[0] = __byte_perm_S (w1[0], w0[3], selector);
+      w2[3] = __byte_perm_S (w0[3], w0[2], selector);
+      w2[2] = __byte_perm_S (w0[2], w0[1], selector);
+      w2[1] = __byte_perm_S (w0[1], w0[0], selector);
+      w2[0] = __byte_perm_S (w0[0],     0, selector);
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 9:
+      w3[1] = __byte_perm_S (w1[0], w0[3], selector);
+      w3[0] = __byte_perm_S (w0[3], w0[2], selector);
+      w2[3] = __byte_perm_S (w0[2], w0[1], selector);
+      w2[2] = __byte_perm_S (w0[1], w0[0], selector);
+      w2[1] = __byte_perm_S (w0[0],     0, selector);
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 10:
+      w3[1] = __byte_perm_S (w0[3], w0[2], selector);
+      w3[0] = __byte_perm_S (w0[2], w0[1], selector);
+      w2[3] = __byte_perm_S (w0[1], w0[0], selector);
+      w2[2] = __byte_perm_S (w0[0],     0, selector);
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 11:
+      w3[1] = __byte_perm_S (w0[2], w0[1], selector);
+      w3[0] = __byte_perm_S (w0[1], w0[0], selector);
+      w2[3] = __byte_perm_S (w0[0],     0, selector);
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 12:
+      w3[1] = __byte_perm_S (w0[1], w0[0], selector);
+      w3[0] = __byte_perm_S (w0[0],     0, selector);
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+
+    case 13:
+      w3[1] = __byte_perm_S (w0[0],     0, selector);
+      w3[0] = 0;
+      w2[3] = 0;
+      w2[2] = 0;
+      w2[1] = 0;
+      w2[0] = 0;
+      w1[3] = 0;
+      w1[2] = 0;
+      w1[1] = 0;
+      w1[0] = 0;
+      w0[3] = 0;
+      w0[2] = 0;
+      w0[1] = 0;
+      w0[0] = 0;
+      break;
+  }
+  #endif
+}
+
+/**
+ * vector functions on scalar types (for inner loop usage)
+ */
+
+#define PACKVS2(sn,vn,e)  \
+  sn[0] = vn[0].s##e;     \
+  sn[1] = vn[1].s##e;
+
+#define PACKSV2(sn,vn,e)  \
+  vn[0].s##e = sn[0];     \
+  vn[1].s##e = sn[1];
+
+#define PACKVS24(s0,s1,v0,v1,e) \
+  PACKVS4 (s0, v0, e);          \
+  PACKVS4 (s1, v1, e);
+
+#define PACKSV24(s0,s1,v0,v1,e) \
+  PACKSV4 (s0, v0, e);          \
+  PACKSV4 (s1, v1, e);
+
+#define PACKVS4(sn,vn,e)  \
+  sn[0] = vn[0].s##e;     \
+  sn[1] = vn[1].s##e;     \
+  sn[2] = vn[2].s##e;     \
+  sn[3] = vn[3].s##e;
+
+#define PACKSV4(sn,vn,e)  \
+  vn[0].s##e = sn[0];     \
+  vn[1].s##e = sn[1];     \
+  vn[2].s##e = sn[2];     \
+  vn[3].s##e = sn[3];
+
+#define PACKVS44(s0,s1,s2,s3,v0,v1,v2,v3,e) \
+  PACKVS4 (s0, v0, e);                      \
+  PACKVS4 (s1, v1, e);                      \
+  PACKVS4 (s2, v2, e);                      \
+  PACKVS4 (s3, v3, e);
+
+#define PACKSV44(s0,s1,s2,s3,v0,v1,v2,v3,e) \
+  PACKSV4 (s0, v0, e);                      \
+  PACKSV4 (s1, v1, e);                      \
+  PACKSV4 (s2, v2, e);                      \
+  PACKSV4 (s3, v3, e);
+
+inline void switch_buffer_by_offset_le_VV (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x offset)
+{
+  #if VECT_SIZE == 1
+
+  switch_buffer_by_offset_le_S (w0, w1, w2, w3, offset);
+
+  #else
+
+  u32 t0[4];
+  u32 t1[4];
+  u32 t2[4];
+  u32 t3[4];
+
+  #endif
+
+  #if   VECT_SIZE == 2
+
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
+
+  #elif VECT_SIZE == 4
+
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
+
+  #elif VECT_SIZE == 8
+
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7);
+
+  #elif VECT_SIZE == 16
+
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s8); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 8);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.s9); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 9);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, a); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sa); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, a);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, b); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sb); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, b);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, c); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sc); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, c);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, d); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sd); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, d);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, e); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.se); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, e);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, f); switch_buffer_by_offset_le_S (t0, t1, t2, t3, offset.sf); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, f);
+
+  #endif
+}
+
+inline void append_0x01_2x4_VV (u32x w0[4], u32x w1[4], const u32x offset)
+{
+  #if VECT_SIZE == 1
+
+  append_0x01_2x4_S (w0, w1, offset);
+
+  #else
+
+  u32 t0[4];
+  u32 t1[4];
+
+  #endif
+
+  #if   VECT_SIZE == 2
+
+  PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
+  PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
+
+  #elif VECT_SIZE == 4
+
+  PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
+  PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
+  PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
+  PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
+
+  #elif VECT_SIZE == 8
+
+  PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
+  PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
+  PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
+  PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
+  PACKVS24 (t0, t1, w0, w1, 4); append_0x01_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4);
+  PACKVS24 (t0, t1, w0, w1, 5); append_0x01_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5);
+  PACKVS24 (t0, t1, w0, w1, 6); append_0x01_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6);
+  PACKVS24 (t0, t1, w0, w1, 7); append_0x01_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7);
+
+  #elif VECT_SIZE == 16
+
+  PACKVS24 (t0, t1, w0, w1, 0); append_0x01_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
+  PACKVS24 (t0, t1, w0, w1, 1); append_0x01_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
+  PACKVS24 (t0, t1, w0, w1, 2); append_0x01_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
+  PACKVS24 (t0, t1, w0, w1, 3); append_0x01_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
+  PACKVS24 (t0, t1, w0, w1, 4); append_0x01_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4);
+  PACKVS24 (t0, t1, w0, w1, 5); append_0x01_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5);
+  PACKVS24 (t0, t1, w0, w1, 6); append_0x01_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6);
+  PACKVS24 (t0, t1, w0, w1, 7); append_0x01_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7);
+  PACKVS24 (t0, t1, w0, w1, 8); append_0x01_2x4_S (t0, t1, offset.s8); PACKSV24 (t0, t1, w0, w1, 8);
+  PACKVS24 (t0, t1, w0, w1, 9); append_0x01_2x4_S (t0, t1, offset.s9); PACKSV24 (t0, t1, w0, w1, 9);
+  PACKVS24 (t0, t1, w0, w1, a); append_0x01_2x4_S (t0, t1, offset.sa); PACKSV24 (t0, t1, w0, w1, a);
+  PACKVS24 (t0, t1, w0, w1, b); append_0x01_2x4_S (t0, t1, offset.sb); PACKSV24 (t0, t1, w0, w1, b);
+  PACKVS24 (t0, t1, w0, w1, c); append_0x01_2x4_S (t0, t1, offset.sc); PACKSV24 (t0, t1, w0, w1, c);
+  PACKVS24 (t0, t1, w0, w1, d); append_0x01_2x4_S (t0, t1, offset.sd); PACKSV24 (t0, t1, w0, w1, d);
+  PACKVS24 (t0, t1, w0, w1, e); append_0x01_2x4_S (t0, t1, offset.se); PACKSV24 (t0, t1, w0, w1, e);
+  PACKVS24 (t0, t1, w0, w1, f); append_0x01_2x4_S (t0, t1, offset.sf); PACKSV24 (t0, t1, w0, w1, f);
+
+  #endif
+}
+
+inline void append_0x80_2x4_VV (u32x w0[4], u32x w1[4], const u32x offset)
+{
+  #if VECT_SIZE == 1
+
+  append_0x80_2x4_S (w0, w1, offset);
+
+  #else
+
+  u32 t0[4];
+  u32 t1[4];
+
+  #endif
+
+  #if   VECT_SIZE == 2
+
+  PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
+  PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
+
+  #elif VECT_SIZE == 4
+
+  PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
+  PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
+  PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
+  PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
+
+  #elif VECT_SIZE == 8
+
+  PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
+  PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
+  PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
+  PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
+  PACKVS24 (t0, t1, w0, w1, 4); append_0x80_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4);
+  PACKVS24 (t0, t1, w0, w1, 5); append_0x80_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5);
+  PACKVS24 (t0, t1, w0, w1, 6); append_0x80_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6);
+  PACKVS24 (t0, t1, w0, w1, 7); append_0x80_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7);
+
+  #elif VECT_SIZE == 16
+
+  PACKVS24 (t0, t1, w0, w1, 0); append_0x80_2x4_S (t0, t1, offset.s0); PACKSV24 (t0, t1, w0, w1, 0);
+  PACKVS24 (t0, t1, w0, w1, 1); append_0x80_2x4_S (t0, t1, offset.s1); PACKSV24 (t0, t1, w0, w1, 1);
+  PACKVS24 (t0, t1, w0, w1, 2); append_0x80_2x4_S (t0, t1, offset.s2); PACKSV24 (t0, t1, w0, w1, 2);
+  PACKVS24 (t0, t1, w0, w1, 3); append_0x80_2x4_S (t0, t1, offset.s3); PACKSV24 (t0, t1, w0, w1, 3);
+  PACKVS24 (t0, t1, w0, w1, 4); append_0x80_2x4_S (t0, t1, offset.s4); PACKSV24 (t0, t1, w0, w1, 4);
+  PACKVS24 (t0, t1, w0, w1, 5); append_0x80_2x4_S (t0, t1, offset.s5); PACKSV24 (t0, t1, w0, w1, 5);
+  PACKVS24 (t0, t1, w0, w1, 6); append_0x80_2x4_S (t0, t1, offset.s6); PACKSV24 (t0, t1, w0, w1, 6);
+  PACKVS24 (t0, t1, w0, w1, 7); append_0x80_2x4_S (t0, t1, offset.s7); PACKSV24 (t0, t1, w0, w1, 7);
+  PACKVS24 (t0, t1, w0, w1, 8); append_0x80_2x4_S (t0, t1, offset.s8); PACKSV24 (t0, t1, w0, w1, 8);
+  PACKVS24 (t0, t1, w0, w1, 9); append_0x80_2x4_S (t0, t1, offset.s9); PACKSV24 (t0, t1, w0, w1, 9);
+  PACKVS24 (t0, t1, w0, w1, a); append_0x80_2x4_S (t0, t1, offset.sa); PACKSV24 (t0, t1, w0, w1, a);
+  PACKVS24 (t0, t1, w0, w1, b); append_0x80_2x4_S (t0, t1, offset.sb); PACKSV24 (t0, t1, w0, w1, b);
+  PACKVS24 (t0, t1, w0, w1, c); append_0x80_2x4_S (t0, t1, offset.sc); PACKSV24 (t0, t1, w0, w1, c);
+  PACKVS24 (t0, t1, w0, w1, d); append_0x80_2x4_S (t0, t1, offset.sd); PACKSV24 (t0, t1, w0, w1, d);
+  PACKVS24 (t0, t1, w0, w1, e); append_0x80_2x4_S (t0, t1, offset.se); PACKSV24 (t0, t1, w0, w1, e);
+  PACKVS24 (t0, t1, w0, w1, f); append_0x80_2x4_S (t0, t1, offset.sf); PACKSV24 (t0, t1, w0, w1, f);
+
+  #endif
+}
+
+inline void append_0x80_4x4_VV (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], const u32x offset)
+{
+  #if VECT_SIZE == 1
+
+  append_0x80_4x4_S (w0, w1, w2, w3, offset);
+
+  #else
+
+  u32 t0[4];
+  u32 t1[4];
+  u32 t2[4];
+  u32 t3[4];
+
+  #endif
+
+  #if   VECT_SIZE == 2
+
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
+
+  #elif VECT_SIZE == 4
+
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
+
+  #elif VECT_SIZE == 8
+
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); append_0x80_4x4_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); append_0x80_4x4_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); append_0x80_4x4_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); append_0x80_4x4_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7);
+
+  #elif VECT_SIZE == 16
+
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 0); append_0x80_4x4_S (t0, t1, t2, t3, offset.s0); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 0);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 1); append_0x80_4x4_S (t0, t1, t2, t3, offset.s1); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 1);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 2); append_0x80_4x4_S (t0, t1, t2, t3, offset.s2); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 2);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 3); append_0x80_4x4_S (t0, t1, t2, t3, offset.s3); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 3);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 4); append_0x80_4x4_S (t0, t1, t2, t3, offset.s4); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 4);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 5); append_0x80_4x4_S (t0, t1, t2, t3, offset.s5); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 5);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 6); append_0x80_4x4_S (t0, t1, t2, t3, offset.s6); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 6);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 7); append_0x80_4x4_S (t0, t1, t2, t3, offset.s7); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 7);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 8); append_0x80_4x4_S (t0, t1, t2, t3, offset.s8); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 8);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, 9); append_0x80_4x4_S (t0, t1, t2, t3, offset.s9); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, 9);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, a); append_0x80_4x4_S (t0, t1, t2, t3, offset.sa); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, a);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, b); append_0x80_4x4_S (t0, t1, t2, t3, offset.sb); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, b);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, c); append_0x80_4x4_S (t0, t1, t2, t3, offset.sc); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, c);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, d); append_0x80_4x4_S (t0, t1, t2, t3, offset.sd); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, d);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, e); append_0x80_4x4_S (t0, t1, t2, t3, offset.se); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, e);
+  PACKVS44 (t0, t1, t2, t3, w0, w1, w2, w3, f); append_0x80_4x4_S (t0, t1, t2, t3, offset.sf); PACKSV44 (t0, t1, t2, t3, w0, w1, w2, w3, f);
+
+  #endif
+}

diff --git a/OpenCL/inc_comp_multi.cl b/OpenCL/inc_comp_multi.cl
new file mode 100644 (file)
index 0000000..14ad55c
--- /dev/null
+++ b/OpenCL/inc_comp_multi.cl
@@ -0,0 +1,32 @@
+u32 digest_tp[4];
+
+digest_tp[0] = r0;
+digest_tp[1] = r1;
+digest_tp[2] = r2;
+digest_tp[3] = r3;
+
+if (check (digest_tp,
+             bitmaps_buf_s1_a,
+             bitmaps_buf_s1_b,
+             bitmaps_buf_s1_c,
+             bitmaps_buf_s1_d,
+             bitmaps_buf_s2_a,
+             bitmaps_buf_s2_b,
+             bitmaps_buf_s2_c,
+             bitmaps_buf_s2_d,
+             bitmap_mask,
+             bitmap_shift1,
+             bitmap_shift2))
+{
+  int digest_pos = find_hash (digest_tp, digests_cnt, &digests_buf[digests_offset]);
+
+  if (digest_pos != -1)
+  {
+    const u32 final_hash_pos = digests_offset + digest_pos;
+
+    if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)
+    {
+      mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos);
+    }
+  }
+}

diff --git a/OpenCL/inc_comp_multi_bs.cl b/OpenCL/inc_comp_multi_bs.cl
new file mode 100644 (file)
index 0000000..6546a97
--- /dev/null
+++ b/OpenCL/inc_comp_multi_bs.cl
@@ -0,0 +1,32 @@
+u32 digest_tp[4];
+
+digest_tp[0] = r0;
+digest_tp[1] = r1;
+digest_tp[2] = r2;
+digest_tp[3] = r3;
+
+if (check (digest_tp,
+             bitmaps_buf_s1_a,
+             bitmaps_buf_s1_b,
+             bitmaps_buf_s1_c,
+             bitmaps_buf_s1_d,
+             bitmaps_buf_s2_a,
+             bitmaps_buf_s2_b,
+             bitmaps_buf_s2_c,
+             bitmaps_buf_s2_d,
+             bitmap_mask,
+             bitmap_shift1,
+             bitmap_shift2))
+{
+  int digest_pos = find_hash (digest_tp, digests_cnt, &digests_buf[digests_offset]);
+
+  if (digest_pos != -1)
+  {
+    const u32 final_hash_pos = digests_offset + digest_pos;
+
+    if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)
+    {
+      mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + slice);
+    }
+  }
+}

diff --git a/OpenCL/inc_comp_single.cl b/OpenCL/inc_comp_single.cl
new file mode 100644 (file)
index 0000000..631a8bc
--- /dev/null
+++ b/OpenCL/inc_comp_single.cl
@@ -0,0 +1,12 @@
+if ((r0 == search[0])
+ && (r1 == search[1])
+ && (r2 == search[2])
+ && (r3 == search[3]))
+{
+  const u32 final_hash_pos = digests_offset + 0;
+
+  if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)
+  {
+    mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos);
+  }
+}

diff --git a/OpenCL/inc_comp_single_bs.cl b/OpenCL/inc_comp_single_bs.cl
new file mode 100644 (file)
index 0000000..5c081c1
--- /dev/null
+++ b/OpenCL/inc_comp_single_bs.cl
@@ -0,0 +1,6 @@
+const u32 final_hash_pos = digests_offset + 0;
+
+if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)
+{
+  mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos);
+}

diff --git a/OpenCL/inc_hash_constants.h b/OpenCL/inc_hash_constants.h
new file mode 100644 (file)
index 0000000..5351445
--- /dev/null
+++ b/OpenCL/inc_hash_constants.h
@@ -0,0 +1,758 @@
+/**
+ * Author......: Jens Steube <jens.steube@gmail.com>
+ * License.....: MIT
+ */
+
+#define COMBINATOR_MODE_BASE_LEFT  10001
+#define COMBINATOR_MODE_BASE_RIGHT 10002
+
+#ifdef SHARED_H
+#define _BCRYPT_
+#define _SHA1_
+#define _SHA256_
+#define _SHA384_
+#define _SHA512_
+#define _MD4_
+#define _MD5_
+#define _MD5H_
+#define _KECCAK_
+#define _RIPEMD160_
+#define _WHIRLPOOL_
+#define _SAPB_
+#define _CLOUDKEY_
+#define _OFFICE2013_
+#define _SIPHASH_
+#define _PDF17L8_
+#define _PBKDF2_MD5_
+#define _PBKDF2_SHA1_
+#define _PBKDF2_SHA256_
+#define _PBKDF2_SHA512_
+#define _RAR3_
+#define _ZIP2_
+#endif
+
+#ifdef _SIPHASH_
+/**
+ * SipHash Constants
+ */
+
+#define SIPHASHM_0 0x736f6d6570736575
+#define SIPHASHM_1 0x646f72616e646f6d
+#define SIPHASHM_2 0x6c7967656e657261
+#define SIPHASHM_3 0x7465646279746573
+#endif
+
+#if defined _BCRYPT_ || defined _PSAFE2_
+/**
+ * bcrypt Constants
+ */
+
+#define BCRYPTM_0 0x4F727068u
+#define BCRYPTM_1 0x65616E42u
+#define BCRYPTM_2 0x65686F6Cu
+#define BCRYPTM_3 0x64657253u
+#define BCRYPTM_4 0x63727944u
+#define BCRYPTM_5 0x6F756274u
+#endif
+
+#if defined _SHA1_ || defined _SAPG_ || defined _OFFICE2007_ || defined _OFFICE2010_ || defined _OLDOFFICE34_ || defined _ANDROIDFDE_ || defined _DCC2_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _PSAFE2_ || defined _LOTUS8_ || defined _PBKDF2_SHA1_ || defined _RAR3_ || defined _SHA256_SHA1_ || defined _ZIP2_
+/**
+ * SHA1 Constants
+ */
+
+#define SHA1M_A 0x67452301u
+#define SHA1M_B 0xefcdab89u
+#define SHA1M_C 0x98badcfeu
+#define SHA1M_D 0x10325476u
+#define SHA1M_E 0xc3d2e1f0u
+
+#define SHA1C00 0x5a827999u
+#define SHA1C01 0x6ed9eba1u
+#define SHA1C02 0x8f1bbcdcu
+#define SHA1C03 0xca62c1d6u
+#endif
+
+#if defined _SHA256_ || defined _PDF17L8_ || defined _SEVEN_ZIP_ || defined _ANDROIDFDE_ || defined _CLOUDKEY_ || defined _SCRYPT_ || defined _PBKDF2_SHA256_ || defined _SHA256_SHA1_ || defined _MS_DRSR_ || defined _ANDROIDFDE_SAMSUNG_ || defined _RAR5_ || defined _KEEPASS_
+/**
+ * SHA256 Constants
+ */
+
+#define SHA256M_A 0x6a09e667u
+#define SHA256M_B 0xbb67ae85u
+#define SHA256M_C 0x3c6ef372u
+#define SHA256M_D 0xa54ff53au
+#define SHA256M_E 0x510e527fu
+#define SHA256M_F 0x9b05688cu
+#define SHA256M_G 0x1f83d9abu
+#define SHA256M_H 0x5be0cd19u
+
+#define SHA256C00 0x428a2f98u
+#define SHA256C01 0x71374491u
+#define SHA256C02 0xb5c0fbcfu
+#define SHA256C03 0xe9b5dba5u
+#define SHA256C04 0x3956c25bu
+#define SHA256C05 0x59f111f1u
+#define SHA256C06 0x923f82a4u
+#define SHA256C07 0xab1c5ed5u
+#define SHA256C08 0xd807aa98u
+#define SHA256C09 0x12835b01u
+#define SHA256C0a 0x243185beu
+#define SHA256C0b 0x550c7dc3u
+#define SHA256C0c 0x72be5d74u
+#define SHA256C0d 0x80deb1feu
+#define SHA256C0e 0x9bdc06a7u
+#define SHA256C0f 0xc19bf174u
+#define SHA256C10 0xe49b69c1u
+#define SHA256C11 0xefbe4786u
+#define SHA256C12 0x0fc19dc6u
+#define SHA256C13 0x240ca1ccu
+#define SHA256C14 0x2de92c6fu
+#define SHA256C15 0x4a7484aau
+#define SHA256C16 0x5cb0a9dcu
+#define SHA256C17 0x76f988dau
+#define SHA256C18 0x983e5152u
+#define SHA256C19 0xa831c66du
+#define SHA256C1a 0xb00327c8u
+#define SHA256C1b 0xbf597fc7u
+#define SHA256C1c 0xc6e00bf3u
+#define SHA256C1d 0xd5a79147u
+#define SHA256C1e 0x06ca6351u
+#define SHA256C1f 0x14292967u
+#define SHA256C20 0x27b70a85u
+#define SHA256C21 0x2e1b2138u
+#define SHA256C22 0x4d2c6dfcu
+#define SHA256C23 0x53380d13u
+#define SHA256C24 0x650a7354u
+#define SHA256C25 0x766a0abbu
+#define SHA256C26 0x81c2c92eu
+#define SHA256C27 0x92722c85u
+#define SHA256C28 0xa2bfe8a1u
+#define SHA256C29 0xa81a664bu
+#define SHA256C2a 0xc24b8b70u
+#define SHA256C2b 0xc76c51a3u
+#define SHA256C2c 0xd192e819u
+#define SHA256C2d 0xd6990624u
+#define SHA256C2e 0xf40e3585u
+#define SHA256C2f 0x106aa070u
+#define SHA256C30 0x19a4c116u
+#define SHA256C31 0x1e376c08u
+#define SHA256C32 0x2748774cu
+#define SHA256C33 0x34b0bcb5u
+#define SHA256C34 0x391c0cb3u
+#define SHA256C35 0x4ed8aa4au
+#define SHA256C36 0x5b9cca4fu
+#define SHA256C37 0x682e6ff3u
+#define SHA256C38 0x748f82eeu
+#define SHA256C39 0x78a5636fu
+#define SHA256C3a 0x84c87814u
+#define SHA256C3b 0x8cc70208u
+#define SHA256C3c 0x90befffau
+#define SHA256C3d 0xa4506cebu
+#define SHA256C3e 0xbef9a3f7u
+#define SHA256C3f 0xc67178f2u
+#endif
+
+#if defined _MD4_ || defined _DCC2_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _MS_DRSR_ || defined _KRB5TGS_
+/**
+ * MD4 Constants
+ */
+
+#define MD4M_A 0x67452301u
+#define MD4M_B 0xefcdab89u
+#define MD4M_C 0x98badcfeu
+#define MD4M_D 0x10325476u
+
+#define MD4S00  3u
+#define MD4S01  7u
+#define MD4S02 11u
+#define MD4S03 19u
+#define MD4S10  3u
+#define MD4S11  5u
+#define MD4S12  9u
+#define MD4S13 13u
+#define MD4S20  3u
+#define MD4S21  9u
+#define MD4S22 11u
+#define MD4S23 15u
+
+#define MD4C00 0x00000000u
+#define MD4C01 0x5a827999u
+#define MD4C02 0x6ed9eba1u
+#endif
+
+#if defined _MD5_ || defined _MD5H_ || defined _SAPB_ || defined _OLDOFFICE01_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _NETNTLMV2_ || defined _KRB5PA_  || defined _PBKDF2_MD5_ || defined _KRB5TGS_
+/**
+ * MD5 Constants
+ */
+
+#define MD5M_A 0x67452301u
+#define MD5M_B 0xefcdab89u
+#define MD5M_C 0x98badcfeu
+#define MD5M_D 0x10325476u
+
+#define MD5S00  7u
+#define MD5S01 12u
+#define MD5S02 17u
+#define MD5S03 22u
+#define MD5S10  5u
+#define MD5S11  9u
+#define MD5S12 14u
+#define MD5S13 20u
+#define MD5S20  4u
+#define MD5S21 11u
+#define MD5S22 16u
+#define MD5S23 23u
+#define MD5S30  6u
+#define MD5S31 10u
+#define MD5S32 15u
+#define MD5S33 21u
+
+#define MD5C00 0xd76aa478u
+#define MD5C01 0xe8c7b756u
+#define MD5C02 0x242070dbu
+#define MD5C03 0xc1bdceeeu
+#define MD5C04 0xf57c0fafu
+#define MD5C05 0x4787c62au
+#define MD5C06 0xa8304613u
+#define MD5C07 0xfd469501u
+#define MD5C08 0x698098d8u
+#define MD5C09 0x8b44f7afu
+#define MD5C0a 0xffff5bb1u
+#define MD5C0b 0x895cd7beu
+#define MD5C0c 0x6b901122u
+#define MD5C0d 0xfd987193u
+#define MD5C0e 0xa679438eu
+#define MD5C0f 0x49b40821u
+#define MD5C10 0xf61e2562u
+#define MD5C11 0xc040b340u
+#define MD5C12 0x265e5a51u
+#define MD5C13 0xe9b6c7aau
+#define MD5C14 0xd62f105du
+#define MD5C15 0x02441453u
+#define MD5C16 0xd8a1e681u
+#define MD5C17 0xe7d3fbc8u
+#define MD5C18 0x21e1cde6u
+#define MD5C19 0xc33707d6u
+#define MD5C1a 0xf4d50d87u
+#define MD5C1b 0x455a14edu
+#define MD5C1c 0xa9e3e905u
+#define MD5C1d 0xfcefa3f8u
+#define MD5C1e 0x676f02d9u
+#define MD5C1f 0x8d2a4c8au
+#define MD5C20 0xfffa3942u
+#define MD5C21 0x8771f681u
+#define MD5C22 0x6d9d6122u
+#define MD5C23 0xfde5380cu
+#define MD5C24 0xa4beea44u
+#define MD5C25 0x4bdecfa9u
+#define MD5C26 0xf6bb4b60u
+#define MD5C27 0xbebfbc70u
+#define MD5C28 0x289b7ec6u
+#define MD5C29 0xeaa127fau
+#define MD5C2a 0xd4ef3085u
+#define MD5C2b 0x04881d05u
+#define MD5C2c 0xd9d4d039u
+#define MD5C2d 0xe6db99e5u
+#define MD5C2e 0x1fa27cf8u
+#define MD5C2f 0xc4ac5665u
+#define MD5C30 0xf4292244u
+#define MD5C31 0x432aff97u
+#define MD5C32 0xab9423a7u
+#define MD5C33 0xfc93a039u
+#define MD5C34 0x655b59c3u
+#define MD5C35 0x8f0ccc92u
+#define MD5C36 0xffeff47du
+#define MD5C37 0x85845dd1u
+#define MD5C38 0x6fa87e4fu
+#define MD5C39 0xfe2ce6e0u
+#define MD5C3a 0xa3014314u
+#define MD5C3b 0x4e0811a1u
+#define MD5C3c 0xf7537e82u
+#define MD5C3d 0xbd3af235u
+#define MD5C3e 0x2ad7d2bbu
+#define MD5C3f 0xeb86d391u
+#endif
+
+#if defined _SHA384_ || defined _PDF17L8_
+/**
+ * SHA384 Constants (64 bits)
+ */
+
+#define SHA384M_A 0xcbbb9d5dc1059ed8
+#define SHA384M_B 0x629a292a367cd507
+#define SHA384M_C 0x9159015a3070dd17
+#define SHA384M_D 0x152fecd8f70e5939
+#define SHA384M_E 0x67332667ffc00b31
+#define SHA384M_F 0x8eb44a8768581511
+#define SHA384M_G 0xdb0c2e0d64f98fa7
+#define SHA384M_H 0x47b5481dbefa4fa4
+
+#define SHA384C00 0x428a2f98d728ae22
+#define SHA384C01 0x7137449123ef65cd
+#define SHA384C02 0xb5c0fbcfec4d3b2f
+#define SHA384C03 0xe9b5dba58189dbbc
+#define SHA384C04 0x3956c25bf348b538
+#define SHA384C05 0x59f111f1b605d019
+#define SHA384C06 0x923f82a4af194f9b
+#define SHA384C07 0xab1c5ed5da6d8118
+#define SHA384C08 0xd807aa98a3030242
+#define SHA384C09 0x12835b0145706fbe
+#define SHA384C0a 0x243185be4ee4b28c
+#define SHA384C0b 0x550c7dc3d5ffb4e2
+#define SHA384C0c 0x72be5d74f27b896f
+#define SHA384C0d 0x80deb1fe3b1696b1
+#define SHA384C0e 0x9bdc06a725c71235
+#define SHA384C0f 0xc19bf174cf692694
+#define SHA384C10 0xe49b69c19ef14ad2
+#define SHA384C11 0xefbe4786384f25e3
+#define SHA384C12 0x0fc19dc68b8cd5b5
+#define SHA384C13 0x240ca1cc77ac9c65
+#define SHA384C14 0x2de92c6f592b0275
+#define SHA384C15 0x4a7484aa6ea6e483
+#define SHA384C16 0x5cb0a9dcbd41fbd4
+#define SHA384C17 0x76f988da831153b5
+#define SHA384C18 0x983e5152ee66dfab
+#define SHA384C19 0xa831c66d2db43210
+#define SHA384C1a 0xb00327c898fb213f
+#define SHA384C1b 0xbf597fc7beef0ee4
+#define SHA384C1c 0xc6e00bf33da88fc2
+#define SHA384C1d 0xd5a79147930aa725
+#define SHA384C1e 0x06ca6351e003826f
+#define SHA384C1f 0x142929670a0e6e70
+#define SHA384C20 0x27b70a8546d22ffc
+#define SHA384C21 0x2e1b21385c26c926
+#define SHA384C22 0x4d2c6dfc5ac42aed
+#define SHA384C23 0x53380d139d95b3df
+#define SHA384C24 0x650a73548baf63de
+#define SHA384C25 0x766a0abb3c77b2a8
+#define SHA384C26 0x81c2c92e47edaee6
+#define SHA384C27 0x92722c851482353b
+#define SHA384C28 0xa2bfe8a14cf10364
+#define SHA384C29 0xa81a664bbc423001
+#define SHA384C2a 0xc24b8b70d0f89791
+#define SHA384C2b 0xc76c51a30654be30
+#define SHA384C2c 0xd192e819d6ef5218
+#define SHA384C2d 0xd69906245565a910
+#define SHA384C2e 0xf40e35855771202a
+#define SHA384C2f 0x106aa07032bbd1b8
+#define SHA384C30 0x19a4c116b8d2d0c8
+#define SHA384C31 0x1e376c085141ab53
+#define SHA384C32 0x2748774cdf8eeb99
+#define SHA384C33 0x34b0bcb5e19b48a8
+#define SHA384C34 0x391c0cb3c5c95a63
+#define SHA384C35 0x4ed8aa4ae3418acb
+#define SHA384C36 0x5b9cca4f7763e373
+#define SHA384C37 0x682e6ff3d6b2b8a3
+#define SHA384C38 0x748f82ee5defb2fc
+#define SHA384C39 0x78a5636f43172f60
+#define SHA384C3a 0x84c87814a1f0ab72
+#define SHA384C3b 0x8cc702081a6439ec
+#define SHA384C3c 0x90befffa23631e28
+#define SHA384C3d 0xa4506cebde82bde9
+#define SHA384C3e 0xbef9a3f7b2c67915
+#define SHA384C3f 0xc67178f2e372532b
+#define SHA384C40 0xca273eceea26619c
+#define SHA384C41 0xd186b8c721c0c207
+#define SHA384C42 0xeada7dd6cde0eb1e
+#define SHA384C43 0xf57d4f7fee6ed178
+#define SHA384C44 0x06f067aa72176fba
+#define SHA384C45 0x0a637dc5a2c898a6
+#define SHA384C46 0x113f9804bef90dae
+#define SHA384C47 0x1b710b35131c471b
+#define SHA384C48 0x28db77f523047d84
+#define SHA384C49 0x32caab7b40c72493
+#define SHA384C4a 0x3c9ebe0a15c9bebc
+#define SHA384C4b 0x431d67c49c100d4c
+#define SHA384C4c 0x4cc5d4becb3e42b6
+#define SHA384C4d 0x597f299cfc657e2a
+#define SHA384C4e 0x5fcb6fab3ad6faec
+#define SHA384C4f 0x6c44198c4a475817
+
+#endif
+
+#if defined _SHA512_ || defined _CLOUDKEY_ || defined _OFFICE2013_ || defined _PDF17L8_ || defined _PBKDF2_SHA512_
+/**
+ * SHA512 Constants (64 bits)
+ */
+
+#define SHA512M_A 0x6a09e667f3bcc908
+#define SHA512M_B 0xbb67ae8584caa73b
+#define SHA512M_C 0x3c6ef372fe94f82b
+#define SHA512M_D 0xa54ff53a5f1d36f1
+#define SHA512M_E 0x510e527fade682d1
+#define SHA512M_F 0x9b05688c2b3e6c1f
+#define SHA512M_G 0x1f83d9abfb41bd6b
+#define SHA512M_H 0x5be0cd19137e2179
+
+#define SHA512C00 0x428a2f98d728ae22
+#define SHA512C01 0x7137449123ef65cd
+#define SHA512C02 0xb5c0fbcfec4d3b2f
+#define SHA512C03 0xe9b5dba58189dbbc
+#define SHA512C04 0x3956c25bf348b538
+#define SHA512C05 0x59f111f1b605d019
+#define SHA512C06 0x923f82a4af194f9b
+#define SHA512C07 0xab1c5ed5da6d8118
+#define SHA512C08 0xd807aa98a3030242
+#define SHA512C09 0x12835b0145706fbe
+#define SHA512C0a 0x243185be4ee4b28c
+#define SHA512C0b 0x550c7dc3d5ffb4e2
+#define SHA512C0c 0x72be5d74f27b896f
+#define SHA512C0d 0x80deb1fe3b1696b1
+#define SHA512C0e 0x9bdc06a725c71235
+#define SHA512C0f 0xc19bf174cf692694
+#define SHA512C10 0xe49b69c19ef14ad2
+#define SHA512C11 0xefbe4786384f25e3
+#define SHA512C12 0x0fc19dc68b8cd5b5
+#define SHA512C13 0x240ca1cc77ac9c65
+#define SHA512C14 0x2de92c6f592b0275
+#define SHA512C15 0x4a7484aa6ea6e483
+#define SHA512C16 0x5cb0a9dcbd41fbd4
+#define SHA512C17 0x76f988da831153b5
+#define SHA512C18 0x983e5152ee66dfab
+#define SHA512C19 0xa831c66d2db43210
+#define SHA512C1a 0xb00327c898fb213f
+#define SHA512C1b 0xbf597fc7beef0ee4
+#define SHA512C1c 0xc6e00bf33da88fc2
+#define SHA512C1d 0xd5a79147930aa725
+#define SHA512C1e 0x06ca6351e003826f
+#define SHA512C1f 0x142929670a0e6e70
+#define SHA512C20 0x27b70a8546d22ffc
+#define SHA512C21 0x2e1b21385c26c926
+#define SHA512C22 0x4d2c6dfc5ac42aed
+#define SHA512C23 0x53380d139d95b3df
+#define SHA512C24 0x650a73548baf63de
+#define SHA512C25 0x766a0abb3c77b2a8
+#define SHA512C26 0x81c2c92e47edaee6
+#define SHA512C27 0x92722c851482353b
+#define SHA512C28 0xa2bfe8a14cf10364
+#define SHA512C29 0xa81a664bbc423001
+#define SHA512C2a 0xc24b8b70d0f89791
+#define SHA512C2b 0xc76c51a30654be30
+#define SHA512C2c 0xd192e819d6ef5218
+#define SHA512C2d 0xd69906245565a910
+#define SHA512C2e 0xf40e35855771202a
+#define SHA512C2f 0x106aa07032bbd1b8
+#define SHA512C30 0x19a4c116b8d2d0c8
+#define SHA512C31 0x1e376c085141ab53
+#define SHA512C32 0x2748774cdf8eeb99
+#define SHA512C33 0x34b0bcb5e19b48a8
+#define SHA512C34 0x391c0cb3c5c95a63
+#define SHA512C35 0x4ed8aa4ae3418acb
+#define SHA512C36 0x5b9cca4f7763e373
+#define SHA512C37 0x682e6ff3d6b2b8a3
+#define SHA512C38 0x748f82ee5defb2fc
+#define SHA512C39 0x78a5636f43172f60
+#define SHA512C3a 0x84c87814a1f0ab72
+#define SHA512C3b 0x8cc702081a6439ec
+#define SHA512C3c 0x90befffa23631e28
+#define SHA512C3d 0xa4506cebde82bde9
+#define SHA512C3e 0xbef9a3f7b2c67915
+#define SHA512C3f 0xc67178f2e372532b
+#define SHA512C40 0xca273eceea26619c
+#define SHA512C41 0xd186b8c721c0c207
+#define SHA512C42 0xeada7dd6cde0eb1e
+#define SHA512C43 0xf57d4f7fee6ed178
+#define SHA512C44 0x06f067aa72176fba
+#define SHA512C45 0x0a637dc5a2c898a6
+#define SHA512C46 0x113f9804bef90dae
+#define SHA512C47 0x1b710b35131c471b
+#define SHA512C48 0x28db77f523047d84
+#define SHA512C49 0x32caab7b40c72493
+#define SHA512C4a 0x3c9ebe0a15c9bebc
+#define SHA512C4b 0x431d67c49c100d4c
+#define SHA512C4c 0x4cc5d4becb3e42b6
+#define SHA512C4d 0x597f299cfc657e2a
+#define SHA512C4e 0x5fcb6fab3ad6faec
+#define SHA512C4f 0x6c44198c4a475817
+
+#define SHA512REV0 0x5218a97a1b97e8a0
+#define SHA512REV1 0x4334c1bea164f555
+
+#endif
+
+#ifdef _RIPEMD160_
+/**
+ * RIPEMD160 Constants
+ */
+
+#define RIPEMD160M_A 0x67452301u
+#define RIPEMD160M_B 0xefcdab89u
+#define RIPEMD160M_C 0x98badcfeu
+#define RIPEMD160M_D 0x10325476u
+#define RIPEMD160M_E 0xc3d2e1f0u
+
+#define RIPEMD160C00 0x00000000u
+#define RIPEMD160C10 0x5a827999u
+#define RIPEMD160C20 0x6ed9eba1u
+#define RIPEMD160C30 0x8f1bbcdcu
+#define RIPEMD160C40 0xa953fd4eu
+#define RIPEMD160C50 0x50a28be6u
+#define RIPEMD160C60 0x5c4dd124u
+#define RIPEMD160C70 0x6d703ef3u
+#define RIPEMD160C80 0x7a6d76e9u
+#define RIPEMD160C90 0x00000000u
+
+#define RIPEMD160S00 11u
+#define RIPEMD160S01 14u
+#define RIPEMD160S02 15u
+#define RIPEMD160S03 12u
+#define RIPEMD160S04  5u
+#define RIPEMD160S05  8u
+#define RIPEMD160S06  7u
+#define RIPEMD160S07  9u
+#define RIPEMD160S08 11u
+#define RIPEMD160S09 13u
+#define RIPEMD160S0A 14u
+#define RIPEMD160S0B 15u
+#define RIPEMD160S0C  6u
+#define RIPEMD160S0D  7u
+#define RIPEMD160S0E  9u
+#define RIPEMD160S0F  8u
+
+#define RIPEMD160S10  7u
+#define RIPEMD160S11  6u
+#define RIPEMD160S12  8u
+#define RIPEMD160S13 13u
+#define RIPEMD160S14 11u
+#define RIPEMD160S15  9u
+#define RIPEMD160S16  7u
+#define RIPEMD160S17 15u
+#define RIPEMD160S18  7u
+#define RIPEMD160S19 12u
+#define RIPEMD160S1A 15u
+#define RIPEMD160S1B  9u
+#define RIPEMD160S1C 11u
+#define RIPEMD160S1D  7u
+#define RIPEMD160S1E 13u
+#define RIPEMD160S1F 12u
+
+#define RIPEMD160S20 11u
+#define RIPEMD160S21 13u
+#define RIPEMD160S22  6u
+#define RIPEMD160S23  7u
+#define RIPEMD160S24 14u
+#define RIPEMD160S25  9u
+#define RIPEMD160S26 13u
+#define RIPEMD160S27 15u
+#define RIPEMD160S28 14u
+#define RIPEMD160S29  8u
+#define RIPEMD160S2A 13u
+#define RIPEMD160S2B  6u
+#define RIPEMD160S2C  5u
+#define RIPEMD160S2D 12u
+#define RIPEMD160S2E  7u
+#define RIPEMD160S2F  5u
+
+#define RIPEMD160S30 11u
+#define RIPEMD160S31 12u
+#define RIPEMD160S32 14u
+#define RIPEMD160S33 15u
+#define RIPEMD160S34 14u
+#define RIPEMD160S35 15u
+#define RIPEMD160S36  9u
+#define RIPEMD160S37  8u
+#define RIPEMD160S38  9u
+#define RIPEMD160S39 14u
+#define RIPEMD160S3A  5u
+#define RIPEMD160S3B  6u
+#define RIPEMD160S3C  8u
+#define RIPEMD160S3D  6u
+#define RIPEMD160S3E  5u
+#define RIPEMD160S3F 12u
+
+#define RIPEMD160S40  9u
+#define RIPEMD160S41 15u
+#define RIPEMD160S42  5u
+#define RIPEMD160S43 11u
+#define RIPEMD160S44  6u
+#define RIPEMD160S45  8u
+#define RIPEMD160S46 13u
+#define RIPEMD160S47 12u
+#define RIPEMD160S48  5u
+#define RIPEMD160S49 12u
+#define RIPEMD160S4A 13u
+#define RIPEMD160S4B 14u
+#define RIPEMD160S4C 11u
+#define RIPEMD160S4D  8u
+#define RIPEMD160S4E  5u
+#define RIPEMD160S4F  6u
+
+#define RIPEMD160S50  8u
+#define RIPEMD160S51  9u
+#define RIPEMD160S52  9u
+#define RIPEMD160S53 11u
+#define RIPEMD160S54 13u
+#define RIPEMD160S55 15u
+#define RIPEMD160S56 15u
+#define RIPEMD160S57  5u
+#define RIPEMD160S58  7u
+#define RIPEMD160S59  7u
+#define RIPEMD160S5A  8u
+#define RIPEMD160S5B 11u
+#define RIPEMD160S5C 14u
+#define RIPEMD160S5D 14u
+#define RIPEMD160S5E 12u
+#define RIPEMD160S5F  6u
+
+#define RIPEMD160S60  9u
+#define RIPEMD160S61 13u
+#define RIPEMD160S62 15u
+#define RIPEMD160S63  7u
+#define RIPEMD160S64 12u
+#define RIPEMD160S65  8u
+#define RIPEMD160S66  9u
+#define RIPEMD160S67 11u
+#define RIPEMD160S68  7u
+#define RIPEMD160S69  7u
+#define RIPEMD160S6A 12u
+#define RIPEMD160S6B  7u
+#define RIPEMD160S6C  6u
+#define RIPEMD160S6D 15u
+#define RIPEMD160S6E 13u
+#define RIPEMD160S6F 11u
+
+#define RIPEMD160S70  9u
+#define RIPEMD160S71  7u
+#define RIPEMD160S72 15u
+#define RIPEMD160S73 11u
+#define RIPEMD160S74  8u
+#define RIPEMD160S75  6u
+#define RIPEMD160S76  6u
+#define RIPEMD160S77 14u
+#define RIPEMD160S78 12u
+#define RIPEMD160S79 13u
+#define RIPEMD160S7A  5u
+#define RIPEMD160S7B 14u
+#define RIPEMD160S7C 13u
+#define RIPEMD160S7D 13u
+#define RIPEMD160S7E  7u
+#define RIPEMD160S7F  5u
+
+#define RIPEMD160S80 15u
+#define RIPEMD160S81  5u
+#define RIPEMD160S82  8u
+#define RIPEMD160S83 11u
+#define RIPEMD160S84 14u
+#define RIPEMD160S85 14u
+#define RIPEMD160S86  6u
+#define RIPEMD160S87 14u
+#define RIPEMD160S88  6u
+#define RIPEMD160S89  9u
+#define RIPEMD160S8A 12u
+#define RIPEMD160S8B  9u
+#define RIPEMD160S8C 12u
+#define RIPEMD160S8D  5u
+#define RIPEMD160S8E 15u
+#define RIPEMD160S8F  8u
+
+#define RIPEMD160S90  8u
+#define RIPEMD160S91  5u
+#define RIPEMD160S92 12u
+#define RIPEMD160S93  9u
+#define RIPEMD160S94 12u
+#define RIPEMD160S95  5u
+#define RIPEMD160S96 14u
+#define RIPEMD160S97  6u
+#define RIPEMD160S98  8u
+#define RIPEMD160S99 13u
+#define RIPEMD160S9A  6u
+#define RIPEMD160S9B  5u
+#define RIPEMD160S9C 15u
+#define RIPEMD160S9D 13u
+#define RIPEMD160S9E 11u
+#define RIPEMD160S9F 11u
+
+#endif
+
+#ifdef _KECCAK_
+/**
+ * KECCAK Constants
+ */
+
+#define KECCAK_RNDC_00 0x0000000000000001
+#define KECCAK_RNDC_01 0x0000000000008082
+#define KECCAK_RNDC_02 0x000000000000808a
+#define KECCAK_RNDC_03 0x0000000080008000
+#define KECCAK_RNDC_04 0x000000000000808b
+#define KECCAK_RNDC_05 0x0000000080000001
+#define KECCAK_RNDC_06 0x0000000080008081
+#define KECCAK_RNDC_07 0x0000000000008009
+#define KECCAK_RNDC_08 0x000000000000008a
+#define KECCAK_RNDC_09 0x0000000000000088
+#define KECCAK_RNDC_10 0x0000000080008009
+#define KECCAK_RNDC_11 0x000000008000000a
+#define KECCAK_RNDC_12 0x000000008000808b
+#define KECCAK_RNDC_13 0x000000000000008b
+#define KECCAK_RNDC_14 0x0000000000008089
+#define KECCAK_RNDC_15 0x0000000000008003
+#define KECCAK_RNDC_16 0x0000000000008002
+#define KECCAK_RNDC_17 0x0000000000000080
+#define KECCAK_RNDC_18 0x000000000000800a
+#define KECCAK_RNDC_19 0x000000008000000a
+#define KECCAK_RNDC_20 0x0000000080008081
+#define KECCAK_RNDC_21 0x0000000000008080
+#define KECCAK_RNDC_22 0x0000000080000001
+#define KECCAK_RNDC_23 0x0000000080008008
+
+#define KECCAK_PILN_00 10
+#define KECCAK_PILN_01  7
+#define KECCAK_PILN_02 11
+#define KECCAK_PILN_03 17
+#define KECCAK_PILN_04 18
+#define KECCAK_PILN_05  3
+#define KECCAK_PILN_06  5
+#define KECCAK_PILN_07 16
+#define KECCAK_PILN_08  8
+#define KECCAK_PILN_09 21
+#define KECCAK_PILN_10 24
+#define KECCAK_PILN_11  4
+#define KECCAK_PILN_12 15
+#define KECCAK_PILN_13 23
+#define KECCAK_PILN_14 19
+#define KECCAK_PILN_15 13
+#define KECCAK_PILN_16 12
+#define KECCAK_PILN_17  2
+#define KECCAK_PILN_18 20
+#define KECCAK_PILN_19 14
+#define KECCAK_PILN_20 22
+#define KECCAK_PILN_21  9
+#define KECCAK_PILN_22  6
+#define KECCAK_PILN_23  1
+
+#define KECCAK_ROTC_00  1
+#define KECCAK_ROTC_01  3
+#define KECCAK_ROTC_02  6
+#define KECCAK_ROTC_03 10
+#define KECCAK_ROTC_04 15
+#define KECCAK_ROTC_05 21
+#define KECCAK_ROTC_06 28
+#define KECCAK_ROTC_07 36
+#define KECCAK_ROTC_08 45
+#define KECCAK_ROTC_09 55
+#define KECCAK_ROTC_10  2
+#define KECCAK_ROTC_11 14
+#define KECCAK_ROTC_12 27
+#define KECCAK_ROTC_13 41
+#define KECCAK_ROTC_14 56
+#define KECCAK_ROTC_15  8
+#define KECCAK_ROTC_16 25
+#define KECCAK_ROTC_17 43
+#define KECCAK_ROTC_18 62
+#define KECCAK_ROTC_19 18
+#define KECCAK_ROTC_20 39
+#define KECCAK_ROTC_21 61
+#define KECCAK_ROTC_22 20
+#define KECCAK_ROTC_23 44
+
+
+#endif
+
+#ifdef _MYSQL323_
+/**
+ * MYSQL323 Constants
+ */
+
+#define MYSQL323_A 0x50305735
+#define MYSQL323_B 0x12345671
+
+#endif

diff --git a/OpenCL/inc_hash_functions.cl b/OpenCL/inc_hash_functions.cl
new file mode 100644 (file)
index 0000000..0ccbb25
--- /dev/null
+++ b/OpenCL/inc_hash_functions.cl
@@ -0,0 +1,413 @@
+/**
+ * Author......: Jens Steube <jens.steube@gmail.com>
+ * License.....: MIT
+ */
+
+#if defined _MD4_ || defined _DCC2_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _MS_DRSR_ || defined _KRB5TGS_
+
+#define MD4_F_S(x,y,z)  (((x) & (y)) | ((~(x)) & (z)))
+#define MD4_G_S(x,y,z)  (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
+#define MD4_H_S(x,y,z)  ((x) ^ (y) ^ (z))
+
+#ifdef IS_NV
+#define MD4_F(x,y,z)    (((x) & (y)) | ((~(x)) & (z)))
+#define MD4_G(x,y,z)    (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
+#define MD4_H(x,y,z)    ((x) ^ (y) ^ (z))
+#define MD4_Fo(x,y,z)   (MD4_F((x), (y), (z)))
+#define MD4_Go(x,y,z)   (MD4_G((x), (y), (z)))
+#endif
+
+#ifdef IS_AMD
+#define MD4_F(x,y,z)    (((x) & (y)) | ((~(x)) & (z)))
+#define MD4_G(x,y,z)    (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
+#define MD4_H(x,y,z)    ((x) ^ (y) ^ (z))
+#define MD4_Fo(x,y,z)   (bitselect ((z), (y), (x)))
+#define MD4_Go(x,y,z)   (bitselect ((x), (y), ((x) ^ (z))))
+#endif
+
+#ifdef IS_GENERIC
+#define MD4_F(x,y,z)    (((x) & (y)) | ((~(x)) & (z)))
+#define MD4_G(x,y,z)    (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
+#define MD4_H(x,y,z)    ((x) ^ (y) ^ (z))
+#define MD4_Fo(x,y,z)   (MD4_F((x), (y), (z)))
+#define MD4_Go(x,y,z)   (MD4_G((x), (y), (z)))
+#endif
+
+#define MD4_STEP_S(f,a,b,c,d,x,K,s) \
+{                                   \
+  a += K;                           \
+  a += x;                           \
+  a += f (b, c, d);                 \
+  a  = rotl32_S (a, s);             \
+}
+
+#define MD4_STEP(f,a,b,c,d,x,K,s)   \
+{                                   \
+  a += K;                           \
+  a += x;                           \
+  a += f (b, c, d);                 \
+  a  = rotl32 (a, s);               \
+}
+
+#define MD4_STEP0(f,a,b,c,d,K,s)    \
+{                                   \
+  a += K;                           \
+  a += f (b, c, d);                 \
+  a  = rotl32 (a, s);               \
+}
+
+#endif
+
+#if defined _MD5_ || defined _MD5H_ || defined _SAPB_ || defined _OLDOFFICE01_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _PBKDF2_MD5_ || defined _KRB5TGS_
+
+#define MD5_F_S(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
+#define MD5_G_S(x,y,z)  ((y) ^ ((z) & ((x) ^ (y))))
+#define MD5_H_S(x,y,z)  ((x) ^ (y) ^ (z))
+#define MD5_I_S(x,y,z)  ((y) ^ ((x) | ~(z)))
+
+#ifdef IS_NV
+#define MD5_F(x,y,z)    ((z) ^ ((x) & ((y) ^ (z))))
+#define MD5_G(x,y,z)    ((y) ^ ((z) & ((x) ^ (y))))
+#define MD5_H(x,y,z)    ((x) ^ (y) ^ (z))
+#define MD5_I(x,y,z)    ((y) ^ ((x) | ~(z)))
+#define MD5_Fo(x,y,z)   (MD5_F((x), (y), (z)))
+#define MD5_Go(x,y,z)   (MD5_G((x), (y), (z)))
+#endif
+
+#ifdef IS_AMD
+#define MD5_F(x,y,z)    ((z) ^ ((x) & ((y) ^ (z))))
+#define MD5_G(x,y,z)    ((y) ^ ((z) & ((x) ^ (y))))
+#define MD5_H(x,y,z)    ((x) ^ (y) ^ (z))
+#define MD5_I(x,y,z)    (bitselect (0xffffffffU, (x), (z)) ^ (y))
+#define MD5_Fo(x,y,z)   (bitselect ((z), (y), (x)))
+#define MD5_Go(x,y,z)   (bitselect ((y), (x), (z)))
+#endif
+
+#ifdef IS_GENERIC
+#define MD5_F(x,y,z)    ((z) ^ ((x) & ((y) ^ (z))))
+#define MD5_G(x,y,z)    ((y) ^ ((z) & ((x) ^ (y))))
+#define MD5_H(x,y,z)    ((x) ^ (y) ^ (z))
+#define MD5_I(x,y,z)    ((y) ^ ((x) | ~(z)))
+#define MD5_Fo(x,y,z)   (MD5_F((x), (y), (z)))
+#define MD5_Go(x,y,z)   (MD5_G((x), (y), (z)))
+#endif
+
+#define MD5_STEP_S(f,a,b,c,d,x,K,s) \
+{                                   \
+  a += K;                           \
+  a += x;                           \
+  a += f (b, c, d);                 \
+  a  = rotl32_S (a, s);             \
+  a += b;                           \
+}
+
+#define MD5_STEP(f,a,b,c,d,x,K,s)   \
+{                                   \
+  a += K;                           \
+  a += x;                           \
+  a += f (b, c, d);                 \
+  a  = rotl32 (a, s);               \
+  a += b;                           \
+}
+
+#define MD5_STEP0(f,a,b,c,d,K,s)    \
+{                                   \
+  a += K;                           \
+  a += f (b, c, d);                 \
+  a  = rotl32 (a, s);               \
+  a += b;                           \
+}
+#endif
+
+#if defined _SHA1_ || defined _SAPG_ || defined _OFFICE2007_ || defined _OFFICE2010_ || defined _OLDOFFICE34_ || defined _ANDROIDFDE_ || defined _DCC2_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _PSAFE2_ || defined _LOTUS8_ || defined _PBKDF2_SHA1_ || defined _RAR3_ || defined _SHA256_SHA1_ || defined _ZIP2_
+
+#ifdef IS_NV
+#define SHA1_F0(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
+#define SHA1_F1(x,y,z)  ((x) ^ (y) ^ (z))
+#define SHA1_F2(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
+#define SHA1_F0o(x,y,z) (SHA1_F0 ((x), (y), (z)))
+#define SHA1_F2o(x,y,z) (SHA1_F2 ((x), (y), (z)))
+#endif
+
+#ifdef IS_AMD
+#define SHA1_F0(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
+#define SHA1_F1(x,y,z)  ((x) ^ (y) ^ (z))
+#define SHA1_F2(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
+#define SHA1_F0o(x,y,z) (bitselect ((z), (y), (x)))
+#define SHA1_F2o(x,y,z) (bitselect ((x), (y), ((x) ^ (z))))
+#endif
+
+#ifdef IS_GENERIC
+#define SHA1_F0(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
+#define SHA1_F1(x,y,z)  ((x) ^ (y) ^ (z))
+#define SHA1_F2(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
+#define SHA1_F0o(x,y,z) (SHA1_F0 ((x), (y), (z)))
+#define SHA1_F2o(x,y,z) (SHA1_F2 ((x), (y), (z)))
+#endif
+
+#define SHA1_STEP_S(f,a,b,c,d,e,x)  \
+{                                   \
+  e += K;                           \
+  e += x;                           \
+  e += f (b, c, d);                 \
+  e += rotl32_S (a,  5u);           \
+  b  = rotl32_S (b, 30u);           \
+}
+
+#define SHA1_STEP(f,a,b,c,d,e,x)    \
+{                                   \
+  e += K;                           \
+  e += x;                           \
+  e += f (b, c, d);                 \
+  e += rotl32 (a,  5u);             \
+  b  = rotl32 (b, 30u);             \
+}
+
+#define SHA1_STEP0(f,a,b,c,d,e,x)   \
+{                                   \
+  e += K;                           \
+  e += f (b, c, d);                 \
+  e += rotl32 (a,  5u);             \
+  b  = rotl32 (b, 30u);             \
+}
+
+#define SHA1_STEPX(f,a,b,c,d,e,x)   \
+{                                   \
+  e += x;                           \
+  e += f (b, c, d);                 \
+  e += rotl32 (a,  5u);             \
+  b  = rotl32 (b, 30u);             \
+}
+
+#define SHA1_STEP_PE(f,a,b,c,d,e,x) \
+{                                   \
+  e += x;                           \
+  e += f (b, c, d);                 \
+  e += rotl32 (a,  5u);             \
+}
+
+#define SHA1_STEP_PB(f,a,b,c,d,e,x) \
+{                                   \
+  e += K;                           \
+  b  = rotl32 (b, 30u);             \
+}
+#endif
+
+#if defined _SHA256_ || defined _PDF17L8_ || defined _SEVEN_ZIP_ || defined _ANDROIDFDE_ || defined _CLOUDKEY_ || defined _SCRYPT_ || defined _PBKDF2_SHA256_  || defined _SHA256_SHA1_ || defined _MS_DRSR_ || defined _ANDROIDFDE_SAMSUNG_ || defined _RAR5_ || defined _KEEPASS_
+
+#define SHIFT_RIGHT_32(x,n) ((x) >> (n))
+
+#define SHA256_S0_S(x) (rotl32_S ((x), 25u) ^ rotl32_S ((x), 14u) ^ SHIFT_RIGHT_32 ((x),  3u))
+#define SHA256_S1_S(x) (rotl32_S ((x), 15u) ^ rotl32_S ((x), 13u) ^ SHIFT_RIGHT_32 ((x), 10u))
+#define SHA256_S2_S(x) (rotl32_S ((x), 30u) ^ rotl32_S ((x), 19u) ^ rotl32_S ((x), 10u))
+#define SHA256_S3_S(x) (rotl32_S ((x), 26u) ^ rotl32_S ((x), 21u) ^ rotl32_S ((x),  7u))
+
+#define SHA256_S0(x) (rotl32 ((x), 25u) ^ rotl32 ((x), 14u) ^ SHIFT_RIGHT_32 ((x),  3u))
+#define SHA256_S1(x) (rotl32 ((x), 15u) ^ rotl32 ((x), 13u) ^ SHIFT_RIGHT_32 ((x), 10u))
+#define SHA256_S2(x) (rotl32 ((x), 30u) ^ rotl32 ((x), 19u) ^ rotl32 ((x), 10u))
+#define SHA256_S3(x) (rotl32 ((x), 26u) ^ rotl32 ((x), 21u) ^ rotl32 ((x),  7u))
+
+#ifdef IS_NV
+#define SHA256_F0(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
+#define SHA256_F1(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
+#define SHA256_F0o(x,y,z) (SHA256_F0 ((x), (y), (z)))
+#define SHA256_F1o(x,y,z) (SHA256_F1 ((x), (y), (z)))
+#endif
+
+#ifdef IS_AMD
+#define SHA256_F0(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
+#define SHA256_F1(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
+#define SHA256_F0o(x,y,z) (bitselect ((x), (y), ((x) ^ (z))))
+#define SHA256_F1o(x,y,z) (bitselect ((z), (y), (x)))
+#endif
+
+#ifdef IS_GENERIC
+#define SHA256_F0(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
+#define SHA256_F1(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
+#define SHA256_F0o(x,y,z) (SHA256_F0 ((x), (y), (z)))
+#define SHA256_F1o(x,y,z) (SHA256_F1 ((x), (y), (z)))
+#endif
+
+#define SHA256_STEP_S(F0,F1,a,b,c,d,e,f,g,h,x,K)  \
+{                                                 \
+  h += K;                                         \
+  h += x;                                         \
+  h += SHA256_S3_S (e);                           \
+  h += F1 (e,f,g);                                \
+  d += h;                                         \
+  h += SHA256_S2_S (a);                           \
+  h += F0 (a,b,c);                                \
+}
+
+#define SHA256_EXPAND_S(x,y,z,w) (SHA256_S1_S (x) + y + SHA256_S0_S (z) + w)
+
+#define SHA256_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K)    \
+{                                                 \
+  h += K;                                         \
+  h += x;                                         \
+  h += SHA256_S3 (e);                             \
+  h += F1 (e,f,g);                                \
+  d += h;                                         \
+  h += SHA256_S2 (a);                             \
+  h += F0 (a,b,c);                                \
+}
+
+#define SHA256_EXPAND(x,y,z,w) (SHA256_S1 (x) + y + SHA256_S0 (z) + w)
+
+#endif
+
+#if defined _SHA384_ || defined _PDF17L8_
+
+#define SHIFT_RIGHT_64(x,n) ((x) >> (n))
+
+#define SHA384_S0(x) (rotr64 ((x), 28) ^ rotr64 ((x), 34) ^ rotr64 ((x), 39))
+#define SHA384_S1(x) (rotr64 ((x), 14) ^ rotr64 ((x), 18) ^ rotr64 ((x), 41))
+#define SHA384_S2(x) (rotr64 ((x),  1) ^ rotr64 ((x),  8) ^ SHIFT_RIGHT_64 ((x), 7))
+#define SHA384_S3(x) (rotr64 ((x), 19) ^ rotr64 ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6))
+
+#define SHA384_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z))))
+#define SHA384_F1(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y))))
+
+#ifdef IS_NV
+#define SHA384_F0o(x,y,z) (SHA384_F0 ((x), (y), (z)))
+#define SHA384_F1o(x,y,z) (SHA384_F1 ((x), (y), (z)))
+#endif
+
+#ifdef IS_AMD
+#define SHA384_F0o(x,y,z) (bitselect ((z), (y), (x)))
+#define SHA384_F1o(x,y,z) (bitselect ((x), (y), ((x) ^ (z))))
+#endif
+
+#ifdef IS_GENERIC
+#define SHA384_F0o(x,y,z) (SHA384_F0 ((x), (y), (z)))
+#define SHA384_F1o(x,y,z) (SHA384_F1 ((x), (y), (z)))
+#endif
+
+#define SHA384_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K)  \
+{                                               \
+  h += K;                                       \
+  h += x;                                       \
+  h += SHA384_S1 (e);                           \
+  h += F0 (e, f, g);                            \
+  d += h;                                       \
+  h += SHA384_S0 (a);                           \
+  h += F1 (a, b, c);                            \
+}
+
+#define SHA384_EXPAND(x,y,z,w) (SHA384_S3 (x) + y + SHA384_S2 (z) + w)
+#endif
+
+#if defined _SHA512_ || defined _CLOUDKEY_ || defined _OFFICE2013_ || defined _PDF17L8_ || defined _PBKDF2_SHA512_
+
+#define SHIFT_RIGHT_64(x,n) ((x) >> (n))
+
+#define SHA512_S0_S(x) (rotr64_S ((x), 28) ^ rotr64_S ((x), 34) ^ rotr64_S ((x), 39))
+#define SHA512_S1_S(x) (rotr64_S ((x), 14) ^ rotr64_S ((x), 18) ^ rotr64_S ((x), 41))
+#define SHA512_S2_S(x) (rotr64_S ((x),  1) ^ rotr64_S ((x),  8) ^ SHIFT_RIGHT_64 ((x), 7))
+#define SHA512_S3_S(x) (rotr64_S ((x), 19) ^ rotr64_S ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6))
+
+#define SHA512_S0(x) (rotr64 ((x), 28) ^ rotr64 ((x), 34) ^ rotr64 ((x), 39))
+#define SHA512_S1(x) (rotr64 ((x), 14) ^ rotr64 ((x), 18) ^ rotr64 ((x), 41))
+#define SHA512_S2(x) (rotr64 ((x),  1) ^ rotr64 ((x),  8) ^ SHIFT_RIGHT_64 ((x), 7))
+#define SHA512_S3(x) (rotr64 ((x), 19) ^ rotr64 ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6))
+
+#define SHA512_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z))))
+#define SHA512_F1(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y))))
+
+#ifdef IS_NV
+#define SHA512_F0o(x,y,z) (SHA512_F0 ((x), (y), (z)))
+#define SHA512_F1o(x,y,z) (SHA512_F1 ((x), (y), (z)))
+#endif
+
+#ifdef IS_AMD
+#define SHA512_F0o(x,y,z) (bitselect ((z), (y), (x)))
+#define SHA512_F1o(x,y,z) (bitselect ((x), (y), ((x) ^ (z))))
+#endif
+
+#ifdef IS_GENERIC
+#define SHA512_F0o(x,y,z) (SHA512_F0 ((x), (y), (z)))
+#define SHA512_F1o(x,y,z) (SHA512_F1 ((x), (y), (z)))
+#endif
+
+#define SHA512_STEP_S(F0,F1,a,b,c,d,e,f,g,h,x,K)  \
+{                                                 \
+  h += K;                                         \
+  h += x;                                         \
+  h += SHA512_S1_S (e);                           \
+  h += F0 (e, f, g);                              \
+  d += h;                                         \
+  h += SHA512_S0_S (a);                           \
+  h += F1 (a, b, c);                              \
+}
+
+#define SHA512_EXPAND_S(x,y,z,w) (SHA512_S3_S (x) + y + SHA512_S2_S (z) + w)
+
+#define SHA512_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K)  \
+{                                               \
+  h += K;                                       \
+  h += x;                                       \
+  h += SHA512_S1 (e);                           \
+  h += F0 (e, f, g);                            \
+  d += h;                                       \
+  h += SHA512_S0 (a);                           \
+  h += F1 (a, b, c);                            \
+}
+
+#define SHA512_EXPAND(x,y,z,w) (SHA512_S3 (x) + y + SHA512_S2 (z) + w)
+#endif
+
+#ifdef _RIPEMD160_
+
+#ifdef IS_NV
+#define RIPEMD160_F(x,y,z)    ((x) ^ (y) ^ (z))
+#define RIPEMD160_G(x,y,z)    ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */
+#define RIPEMD160_H(x,y,z)    (((x) | ~(y)) ^ (z))
+#define RIPEMD160_I(x,y,z)    ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */
+#define RIPEMD160_J(x,y,z)    ((x) ^ ((y) | ~(z)))
+#define RIPEMD160_Go(x,y,z)   (RIPEMD160_G ((x), (y), (z)))
+#define RIPEMD160_Io(x,y,z)   (RIPEMD160_I ((x), (y), (z)))
+#endif
+
+#ifdef IS_AMD
+#define RIPEMD160_F(x,y,z)    ((x) ^ (y) ^ (z))
+#define RIPEMD160_G(x,y,z)    ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */
+#define RIPEMD160_H(x,y,z)    (((x) | ~(y)) ^ (z))
+#define RIPEMD160_I(x,y,z)    ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */
+#define RIPEMD160_J(x,y,z)    ((x) ^ ((y) | ~(z)))
+#define RIPEMD160_Go(x,y,z)   (bitselect ((z), (y), (x)))
+#define RIPEMD160_Io(x,y,z)   (bitselect ((y), (x), (z)))
+#endif
+
+#ifdef IS_GENERIC
+#define RIPEMD160_F(x,y,z)    ((x) ^ (y) ^ (z))
+#define RIPEMD160_G(x,y,z)    ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */
+#define RIPEMD160_H(x,y,z)    (((x) | ~(y)) ^ (z))
+#define RIPEMD160_I(x,y,z)    ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */
+#define RIPEMD160_J(x,y,z)    ((x) ^ ((y) | ~(z)))
+#define RIPEMD160_Go(x,y,z)   (RIPEMD160_G ((x), (y), (z)))
+#define RIPEMD160_Io(x,y,z)   (RIPEMD160_I ((x), (y), (z)))
+#endif
+
+#define RIPEMD160_STEP(f,a,b,c,d,e,x,K,s) \
+{                                         \
+  a += K;                                 \
+  a += x;                                 \
+  a += f (b, c, d);                       \
+  a  = rotl32 (a, s);                     \
+  a += e;                                 \
+  c  = rotl32 (c, 10u);                   \
+}
+
+#define ROTATE_LEFT_WORKAROUND_BUG(a,n) ((a << n) | (a >> (32 - n)))
+
+#define RIPEMD160_STEP_WORKAROUND_BUG(f,a,b,c,d,e,x,K,s)  \
+{                                         \
+  a += K;                                 \
+  a += x;                                 \
+  a += f (b, c, d);                       \
+  a  = ROTATE_LEFT_WORKAROUND_BUG (a, s); \
+  a += e;                                 \
+  c  = rotl32 (c, 10u);                   \
+}
+
+#endif

diff --git a/OpenCL/inc_rp.cl b/OpenCL/inc_rp.cl
new file mode 100644 (file)
index 0000000..3a93e19
--- /dev/null
+++ b/OpenCL/inc_rp.cl
@@ -0,0 +1,4357 @@
+/**
+ * Authors.....: Jens Steube <jens.steube@gmail.com>
+ *               magnum <john.magnum@hushmail.com>
+ *
+ * License.....: MIT
+ */
+
+inline u32  apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len);
+inline u32  apply_rules (const __global u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len);
+inline u32x apply_rules_vect (const u32 pw_buf0[4], const u32 pw_buf1[4], const u32 pw_len, const __global kernel_rule_t *rules_buf, const u32 il_pos, u32x w0[4], u32x w1[4]);
+
+inline u32 generate_cmask (u32 buf)
+{
+  const u32 rmask = ((buf & 0x40404040) >> 1)
+                 & ~((buf & 0x80808080) >> 2);
+
+  const u32 hmask = (buf & 0x1f1f1f1f) + 0x05050505;
+  const u32 lmask = (buf & 0x1f1f1f1f) + 0x1f1f1f1f;
+
+  return rmask & ~hmask & lmask;
+}
+
+inline void truncate_right (u32 w0[4], u32 w1[4], const u32 len)
+{
+  const u32 tmp = (1 << ((len % 4) * 8)) - 1;
+
+  switch (len / 4)
+  {
+    case  0:  w0[0] &= tmp;
+              w0[1]  = 0;
+              w0[2]  = 0;
+              w0[3]  = 0;
+              w1[0]  = 0;
+              w1[1]  = 0;
+              w1[2]  = 0;
+              w1[3]  = 0;
+              break;
+    case  1:  w0[1] &= tmp;
+              w0[2]  = 0;
+              w0[3]  = 0;
+              w1[0]  = 0;
+              w1[1]  = 0;
+              w1[2]  = 0;
+              w1[3]  = 0;
+              break;
+    case  2:  w0[2] &= tmp;
+              w0[3]  = 0;
+              w1[0]  = 0;
+              w1[1]  = 0;
+              w1[2]  = 0;
+              w1[3]  = 0;
+              break;
+    case  3:  w0[3] &= tmp;
+              w1[0]  = 0;
+              w1[1]  = 0;
+              w1[2]  = 0;
+              w1[3]  = 0;
+              break;
+    case  4:  w1[0] &= tmp;
+              w1[1]  = 0;
+              w1[2]  = 0;
+              w1[3]  = 0;
+              break;
+    case  5:  w1[1] &= tmp;
+              w1[2]  = 0;
+              w1[3]  = 0;
+              break;
+    case  6:  w1[2] &= tmp;
+              w1[3]  = 0;
+              break;
+    case  7:  w1[3] &= tmp;
+              break;
+  }
+}
+
+inline void truncate_left (u32 w0[4], u32 w1[4], const u32 len)
+{
+  const u32 tmp = ~((1 << ((len % 4) * 8)) - 1);
+
+  switch (len / 4)
+  {
+    case  0:  w0[0] &= tmp;
+              break;
+    case  1:  w0[0]  = 0;
+              w0[1] &= tmp;
+              break;
+    case  2:  w0[0]  = 0;
+              w0[1]  = 0;
+              w0[2] &= tmp;
+              break;
+    case  3:  w0[0]  = 0;
+              w0[1]  = 0;
+              w0[2]  = 0;
+              w0[3] &= tmp;
+              break;
+    case  4:  w0[0]  = 0;
+              w0[1]  = 0;
+              w0[2]  = 0;
+              w0[3]  = 0;
+              w1[0] &= tmp;
+              break;
+    case  5:  w0[0]  = 0;
+              w0[1]  = 0;
+              w0[2]  = 0;
+              w0[3]  = 0;
+              w1[0]  = 0;
+              w1[1] &= tmp;
+              break;
+    case  6:  w0[0]  = 0;
+              w0[1]  = 0;
+              w0[2]  = 0;
+              w0[3]  = 0;
+              w1[0]  = 0;
+              w1[1]  = 0;
+              w1[2] &= tmp;
+              break;
+    case  7:  w0[0]  = 0;
+              w0[1]  = 0;
+              w0[2]  = 0;
+              w0[3]  = 0;
+              w1[0]  = 0;
+              w1[1]  = 0;
+              w1[2]  = 0;
+              w1[3] &= tmp;
+              break;
+  }
+}
+
+inline void lshift_block (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4])
+{
+  #ifdef IS_NV
+  out0[0] = __byte_perm_S (in0[0], in0[1], 0x4321);
+  out0[1] = __byte_perm_S (in0[1], in0[2], 0x4321);
+  out0[2] = __byte_perm_S (in0[2], in0[3], 0x4321);
+  out0[3] = __byte_perm_S (in0[3], in1[0], 0x4321);
+  out1[0] = __byte_perm_S (in1[0], in1[1], 0x4321);
+  out1[1] = __byte_perm_S (in1[1], in1[2], 0x4321);
+  out1[2] = __byte_perm_S (in1[2], in1[3], 0x4321);
+  out1[3] = __byte_perm_S (in1[3],      0, 0x4321);
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  out0[0] = amd_bytealign_S (in0[1], in0[0], 1);
+  out0[1] = amd_bytealign_S (in0[2], in0[1], 1);
+  out0[2] = amd_bytealign_S (in0[3], in0[2], 1);
+  out0[3] = amd_bytealign_S (in1[0], in0[3], 1);
+  out1[0] = amd_bytealign_S (in1[1], in1[0], 1);
+  out1[1] = amd_bytealign_S (in1[2], in1[1], 1);
+  out1[2] = amd_bytealign_S (in1[3], in1[2], 1);
+  out1[3] = amd_bytealign_S (     0, in1[3], 1);
+  #endif
+}
+
+inline void rshift_block (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4])
+{
+  #ifdef IS_NV
+  out1[3] = __byte_perm_S (in1[2], in1[3], 0x6543);
+  out1[2] = __byte_perm_S (in1[1], in1[2], 0x6543);
+  out1[1] = __byte_perm_S (in1[0], in1[1], 0x6543);
+  out1[0] = __byte_perm_S (in0[3], in1[0], 0x6543);
+  out0[3] = __byte_perm_S (in0[2], in0[3], 0x6543);
+  out0[2] = __byte_perm_S (in0[1], in0[2], 0x6543);
+  out0[1] = __byte_perm_S (in0[0], in0[1], 0x6543);
+  out0[0] = __byte_perm_S (     0, in0[0], 0x6543);
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  out1[3] = amd_bytealign_S (in1[3], in1[2], 3);
+  out1[2] = amd_bytealign_S (in1[2], in1[1], 3);
+  out1[1] = amd_bytealign_S (in1[1], in1[0], 3);
+  out1[0] = amd_bytealign_S (in1[0], in0[3], 3);
+  out0[3] = amd_bytealign_S (in0[3], in0[2], 3);
+  out0[2] = amd_bytealign_S (in0[2], in0[1], 3);
+  out0[1] = amd_bytealign_S (in0[1], in0[0], 3);
+  out0[0] = amd_bytealign_S (in0[0],      0, 3);
+  #endif
+}
+
+inline void lshift_block_N (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4], const u32 num)
+{
+  #ifdef IS_NV
+  switch (num)
+  {
+    case  0:  out0[0] = in0[0];
+              out0[1] = in0[1];
+              out0[2] = in0[2];
+              out0[3] = in0[3];
+              out1[0] = in1[0];
+              out1[1] = in1[1];
+              out1[2] = in1[2];
+              out1[3] = in1[3];
+              break;
+    case  1:  out0[0] = __byte_perm_S (in0[0], in0[1], 0x4321);
+              out0[1] = __byte_perm_S (in0[1], in0[2], 0x4321);
+              out0[2] = __byte_perm_S (in0[2], in0[3], 0x4321);
+              out0[3] = __byte_perm_S (in0[3], in1[0], 0x4321);
+              out1[0] = __byte_perm_S (in1[0], in1[1], 0x4321);
+              out1[1] = __byte_perm_S (in1[1], in1[2], 0x4321);
+              out1[2] = __byte_perm_S (in1[2], in1[3], 0x4321);
+              out1[3] = __byte_perm_S (in1[3],      0, 0x4321);
+              break;
+    case  2:  out0[0] = __byte_perm_S (in0[0], in0[1], 0x5432);
+              out0[1] = __byte_perm_S (in0[1], in0[2], 0x5432);
+              out0[2] = __byte_perm_S (in0[2], in0[3], 0x5432);
+              out0[3] = __byte_perm_S (in0[3], in1[0], 0x5432);
+              out1[0] = __byte_perm_S (in1[0], in1[1], 0x5432);
+              out1[1] = __byte_perm_S (in1[1], in1[2], 0x5432);
+              out1[2] = __byte_perm_S (in1[2], in1[3], 0x5432);
+              out1[3] = __byte_perm_S (in1[3],      0, 0x5432);
+              break;
+    case  3:  out0[0] = __byte_perm_S (in0[0], in0[1], 0x6543);
+              out0[1] = __byte_perm_S (in0[1], in0[2], 0x6543);
+              out0[2] = __byte_perm_S (in0[2], in0[3], 0x6543);
+              out0[3] = __byte_perm_S (in0[3], in1[0], 0x6543);
+              out1[0] = __byte_perm_S (in1[0], in1[1], 0x6543);
+              out1[1] = __byte_perm_S (in1[1], in1[2], 0x6543);
+              out1[2] = __byte_perm_S (in1[2], in1[3], 0x6543);
+              out1[3] = __byte_perm_S (in1[3],      0, 0x6543);
+              break;
+    case  4:  out0[0] = in0[1];
+              out0[1] = in0[2];
+              out0[2] = in0[3];
+              out0[3] = in1[0];
+              out1[0] = in1[1];
+              out1[1] = in1[2];
+              out1[2] = in1[3];
+              out1[3] = 0;
+              break;
+    case  5:  out0[0] = __byte_perm_S (in0[1], in0[2], 0x4321);
+              out0[1] = __byte_perm_S (in0[2], in0[3], 0x4321);
+              out0[2] = __byte_perm_S (in0[3], in1[0], 0x4321);
+              out0[3] = __byte_perm_S (in1[0], in1[1], 0x4321);
+              out1[0] = __byte_perm_S (in1[1], in1[2], 0x4321);
+              out1[1] = __byte_perm_S (in1[2], in1[3], 0x4321);
+              out1[2] = __byte_perm_S (in1[3],      0, 0x4321);
+              out1[3] = 0;
+              break;
+    case  6:  out0[0] = __byte_perm_S (in0[1], in0[2], 0x5432);
+              out0[1] = __byte_perm_S (in0[2], in0[3], 0x5432);
+              out0[2] = __byte_perm_S (in0[3], in1[0], 0x5432);
+              out0[3] = __byte_perm_S (in1[0], in1[1], 0x5432);
+              out1[0] = __byte_perm_S (in1[1], in1[2], 0x5432);
+              out1[1] = __byte_perm_S (in1[2], in1[3], 0x5432);
+              out1[2] = __byte_perm_S (in1[3],      0, 0x5432);
+              out1[3] = 0;
+              break;
+    case  7:  out0[0] = __byte_perm_S (in0[1], in0[2], 0x6543);
+              out0[1] = __byte_perm_S (in0[2], in0[3], 0x6543);
+              out0[2] = __byte_perm_S (in0[3], in1[0], 0x6543);
+              out0[3] = __byte_perm_S (in1[0], in1[1], 0x6543);
+              out1[0] = __byte_perm_S (in1[1], in1[2], 0x6543);
+              out1[1] = __byte_perm_S (in1[2], in1[3], 0x6543);
+              out1[2] = __byte_perm_S (in1[3],      0, 0x6543);
+              out1[3] = 0;
+              break;
+    case  8:  out0[0] = in0[2];
+              out0[1] = in0[3];
+              out0[2] = in1[0];
+              out0[3] = in1[1];
+              out1[0] = in1[2];
+              out1[1] = in1[3];
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case  9:  out0[0] = __byte_perm_S (in0[2], in0[3], 0x4321);
+              out0[1] = __byte_perm_S (in0[3], in1[0], 0x4321);
+              out0[2] = __byte_perm_S (in1[0], in1[1], 0x4321);
+              out0[3] = __byte_perm_S (in1[1], in1[2], 0x4321);
+              out1[0] = __byte_perm_S (in1[2], in1[3], 0x4321);
+              out1[1] = __byte_perm_S (in1[3],      0, 0x4321);
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 10:  out0[0] = __byte_perm_S (in0[2], in0[3], 0x5432);
+              out0[1] = __byte_perm_S (in0[3], in1[0], 0x5432);
+              out0[2] = __byte_perm_S (in1[0], in1[1], 0x5432);
+              out0[3] = __byte_perm_S (in1[1], in1[2], 0x5432);
+              out1[0] = __byte_perm_S (in1[2], in1[3], 0x5432);
+              out1[1] = __byte_perm_S (in1[3],      0, 0x5432);
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 11:  out0[0] = __byte_perm_S (in0[2], in0[3], 0x6543);
+              out0[1] = __byte_perm_S (in0[3], in1[0], 0x6543);
+              out0[2] = __byte_perm_S (in1[0], in1[1], 0x6543);
+              out0[3] = __byte_perm_S (in1[1], in1[2], 0x6543);
+              out1[0] = __byte_perm_S (in1[2], in1[3], 0x6543);
+              out1[1] = __byte_perm_S (in1[3],      0, 0x6543);
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 12:  out0[0] = in0[3];
+              out0[1] = in1[0];
+              out0[2] = in1[1];
+              out0[3] = in1[2];
+              out1[0] = in1[3];
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 13:
+              out0[0] = __byte_perm_S (in0[3], in1[0], 0x4321);
+              out0[1] = __byte_perm_S (in1[0], in1[1], 0x4321);
+              out0[2] = __byte_perm_S (in1[1], in1[2], 0x4321);
+              out0[3] = __byte_perm_S (in1[2], in1[3], 0x4321);
+              out1[0] = __byte_perm_S (in1[3],      0, 0x4321);
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 14:  out0[0] = __byte_perm_S (in0[3], in1[0], 0x5432);
+              out0[1] = __byte_perm_S (in1[0], in1[1], 0x5432);
+              out0[2] = __byte_perm_S (in1[1], in1[2], 0x5432);
+              out0[3] = __byte_perm_S (in1[2], in1[3], 0x5432);
+              out1[0] = __byte_perm_S (in1[3],      0, 0x5432);
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 15:  out0[0] = __byte_perm_S (in0[3], in1[0], 0x6543);
+              out0[1] = __byte_perm_S (in1[0], in1[1], 0x6543);
+              out0[2] = __byte_perm_S (in1[1], in1[2], 0x6543);
+              out0[3] = __byte_perm_S (in1[2], in1[3], 0x6543);
+              out1[0] = __byte_perm_S (in1[3],      0, 0x6543);
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 16:  out0[0] = in1[0];
+              out0[1] = in1[1];
+              out0[2] = in1[2];
+              out0[3] = in1[3];
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 17:  out0[0] = __byte_perm_S (in1[0], in1[1], 0x4321);
+              out0[1] = __byte_perm_S (in1[1], in1[2], 0x4321);
+              out0[2] = __byte_perm_S (in1[2], in1[3], 0x4321);
+              out0[3] = __byte_perm_S (in1[3],      0, 0x4321);
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 18:  out0[0] = __byte_perm_S (in1[0], in1[1], 0x5432);
+              out0[1] = __byte_perm_S (in1[1], in1[2], 0x5432);
+              out0[2] = __byte_perm_S (in1[2], in1[3], 0x5432);
+              out0[3] = __byte_perm_S (in1[3],      0, 0x5432);
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 19:  out0[0] = __byte_perm_S (in1[0], in1[1], 0x6543);
+              out0[1] = __byte_perm_S (in1[1], in1[2], 0x6543);
+              out0[2] = __byte_perm_S (in1[2], in1[3], 0x6543);
+              out0[3] = __byte_perm_S (in1[3],      0, 0x6543);
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 20:  out0[0] = in1[1];
+              out0[1] = in1[2];
+              out0[2] = in1[3];
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 21:  out0[0] = __byte_perm_S (in1[1], in1[2], 0x4321);
+              out0[1] = __byte_perm_S (in1[2], in1[3], 0x4321);
+              out0[2] = __byte_perm_S (in1[3],      0, 0x4321);
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 22:  out0[0] = __byte_perm_S (in1[1], in1[2], 0x5432);
+              out0[1] = __byte_perm_S (in1[2], in1[3], 0x5432);
+              out0[2] = __byte_perm_S (in1[3],      0, 0x5432);
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 23:  out0[0] = __byte_perm_S (in1[1], in1[2], 0x6543);
+              out0[1] = __byte_perm_S (in1[2], in1[3], 0x6543);
+              out0[2] = __byte_perm_S (in1[3],      0, 0x6543);
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 24:  out0[0] = in1[2];
+              out0[1] = in1[3];
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 25:  out0[0] = __byte_perm_S (in1[2], in1[3], 0x4321);
+              out0[1] = __byte_perm_S (in1[3],      0, 0x4321);
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 26:  out0[0] = __byte_perm_S (in1[2], in1[3], 0x5432);
+              out0[1] = __byte_perm_S (in1[3],      0, 0x5432);
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 27:  out0[0] = __byte_perm_S (in1[2], in1[3], 0x6543);
+              out0[1] = __byte_perm_S (in1[3],      0, 0x6543);
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 28:  out0[0] = in1[3];
+              out0[1] = 0;
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 29:  out0[0] = __byte_perm_S (in1[3],     0, 0x4321);
+              out0[1] = 0;
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 30:  out0[0] =  __byte_perm_S (in1[3],     0, 0x5432);
+              out0[1] = 0;
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 31:  out0[0] =  __byte_perm_S (in1[3],     0, 0x6543);
+              out0[1] = 0;
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+  }
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  switch (num)
+  {
+    case  0:  out0[0] = in0[0];
+              out0[1] = in0[1];
+              out0[2] = in0[2];
+              out0[3] = in0[3];
+              out1[0] = in1[0];
+              out1[1] = in1[1];
+              out1[2] = in1[2];
+              out1[3] = in1[3];
+              break;
+    case  1:  out0[0] = amd_bytealign_S (in0[1], in0[0], 1);
+              out0[1] = amd_bytealign_S (in0[2], in0[1], 1);
+              out0[2] = amd_bytealign_S (in0[3], in0[2], 1);
+              out0[3] = amd_bytealign_S (in1[0], in0[3], 1);
+              out1[0] = amd_bytealign_S (in1[1], in1[0], 1);
+              out1[1] = amd_bytealign_S (in1[2], in1[1], 1);
+              out1[2] = amd_bytealign_S (in1[3], in1[2], 1);
+              out1[3] = amd_bytealign_S (     0, in1[3], 1);
+              break;
+    case  2:  out0[0] = amd_bytealign_S (in0[1], in0[0], 2);
+              out0[1] = amd_bytealign_S (in0[2], in0[1], 2);
+              out0[2] = amd_bytealign_S (in0[3], in0[2], 2);
+              out0[3] = amd_bytealign_S (in1[0], in0[3], 2);
+              out1[0] = amd_bytealign_S (in1[1], in1[0], 2);
+              out1[1] = amd_bytealign_S (in1[2], in1[1], 2);
+              out1[2] = amd_bytealign_S (in1[3], in1[2], 2);
+              out1[3] = amd_bytealign_S (     0, in1[3], 2);
+              break;
+    case  3:  out0[0] = amd_bytealign_S (in0[1], in0[0], 3);
+              out0[1] = amd_bytealign_S (in0[2], in0[1], 3);
+              out0[2] = amd_bytealign_S (in0[3], in0[2], 3);
+              out0[3] = amd_bytealign_S (in1[0], in0[3], 3);
+              out1[0] = amd_bytealign_S (in1[1], in1[0], 3);
+              out1[1] = amd_bytealign_S (in1[2], in1[1], 3);
+              out1[2] = amd_bytealign_S (in1[3], in1[2], 3);
+              out1[3] = amd_bytealign_S (     0, in1[3], 3);
+              break;
+    case  4:  out0[0] = in0[1];
+              out0[1] = in0[2];
+              out0[2] = in0[3];
+              out0[3] = in1[0];
+              out1[0] = in1[1];
+              out1[1] = in1[2];
+              out1[2] = in1[3];
+              out1[3] = 0;
+              break;
+    case  5:  out0[0] = amd_bytealign_S (in0[2], in0[1], 1);
+              out0[1] = amd_bytealign_S (in0[3], in0[2], 1);
+              out0[2] = amd_bytealign_S (in1[0], in0[3], 1);
+              out0[3] = amd_bytealign_S (in1[1], in1[0], 1);
+              out1[0] = amd_bytealign_S (in1[2], in1[1], 1);
+              out1[1] = amd_bytealign_S (in1[3], in1[2], 1);
+              out1[2] = amd_bytealign_S (     0, in1[3], 1);
+              out1[3] = 0;
+              break;
+    case  6:  out0[0] = amd_bytealign_S (in0[2], in0[1], 2);
+              out0[1] = amd_bytealign_S (in0[3], in0[2], 2);
+              out0[2] = amd_bytealign_S (in1[0], in0[3], 2);
+              out0[3] = amd_bytealign_S (in1[1], in1[0], 2);
+              out1[0] = amd_bytealign_S (in1[2], in1[1], 2);
+              out1[1] = amd_bytealign_S (in1[3], in1[2], 2);
+              out1[2] = amd_bytealign_S (     0, in1[3], 2);
+              out1[3] = 0;
+              break;
+    case  7:  out0[0] = amd_bytealign_S (in0[2], in0[1], 3);
+              out0[1] = amd_bytealign_S (in0[3], in0[2], 3);
+              out0[2] = amd_bytealign_S (in1[0], in0[3], 3);
+              out0[3] = amd_bytealign_S (in1[1], in1[0], 3);
+              out1[0] = amd_bytealign_S (in1[2], in1[1], 3);
+              out1[1] = amd_bytealign_S (in1[3], in1[2], 3);
+              out1[2] = amd_bytealign_S (     0, in1[3], 3);
+              out1[3] = 0;
+              break;
+    case  8:  out0[0] = in0[2];
+              out0[1] = in0[3];
+              out0[2] = in1[0];
+              out0[3] = in1[1];
+              out1[0] = in1[2];
+              out1[1] = in1[3];
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case  9:  out0[0] = amd_bytealign_S (in0[3], in0[2], 1);
+              out0[1] = amd_bytealign_S (in1[0], in0[3], 1);
+              out0[2] = amd_bytealign_S (in1[1], in1[0], 1);
+              out0[3] = amd_bytealign_S (in1[2], in1[1], 1);
+              out1[0] = amd_bytealign_S (in1[3], in1[2], 1);
+              out1[1] = amd_bytealign_S (     0, in1[3], 1);
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 10:  out0[0] = amd_bytealign_S (in0[3], in0[2], 2);
+              out0[1] = amd_bytealign_S (in1[0], in0[3], 2);
+              out0[2] = amd_bytealign_S (in1[1], in1[0], 2);
+              out0[3] = amd_bytealign_S (in1[2], in1[1], 2);
+              out1[0] = amd_bytealign_S (in1[3], in1[2], 2);
+              out1[1] = amd_bytealign_S (     0, in1[3], 2);
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 11:  out0[0] = amd_bytealign_S (in0[3], in0[2], 3);
+              out0[1] = amd_bytealign_S (in1[0], in0[3], 3);
+              out0[2] = amd_bytealign_S (in1[1], in1[0], 3);
+              out0[3] = amd_bytealign_S (in1[2], in1[1], 3);
+              out1[0] = amd_bytealign_S (in1[3], in1[2], 3);
+              out1[1] = amd_bytealign_S (     0, in1[3], 3);
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 12:  out0[0] = in0[3];
+              out0[1] = in1[0];
+              out0[2] = in1[1];
+              out0[3] = in1[2];
+              out1[0] = in1[3];
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 13:  out0[0] = amd_bytealign_S (in1[0], in0[3], 1);
+              out0[1] = amd_bytealign_S (in1[1], in1[0], 1);
+              out0[2] = amd_bytealign_S (in1[2], in1[1], 1);
+              out0[3] = amd_bytealign_S (in1[3], in1[2], 1);
+              out1[0] = amd_bytealign_S (     0, in1[3], 1);
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 14:  out0[0] = amd_bytealign_S (in1[0], in0[3], 2);
+              out0[1] = amd_bytealign_S (in1[1], in1[0], 2);
+              out0[2] = amd_bytealign_S (in1[2], in1[1], 2);
+              out0[3] = amd_bytealign_S (in1[3], in1[2], 2);
+              out1[0] = amd_bytealign_S (     0, in1[3], 2);
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 15:  out0[0] = amd_bytealign_S (in1[0], in0[3], 3);
+              out0[1] = amd_bytealign_S (in1[1], in1[0], 3);
+              out0[2] = amd_bytealign_S (in1[2], in1[1], 3);
+              out0[3] = amd_bytealign_S (in1[3], in1[2], 3);
+              out1[0] = amd_bytealign_S (     0, in1[3], 3);
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 16:  out0[0] = in1[0];
+              out0[1] = in1[1];
+              out0[2] = in1[2];
+              out0[3] = in1[3];
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 17:  out0[0] = amd_bytealign_S (in1[1], in1[0], 1);
+              out0[1] = amd_bytealign_S (in1[2], in1[1], 1);
+              out0[2] = amd_bytealign_S (in1[3], in1[2], 1);
+              out0[3] = amd_bytealign_S (     0, in1[3], 1);
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 18:  out0[0] = amd_bytealign_S (in1[1], in1[0], 2);
+              out0[1] = amd_bytealign_S (in1[2], in1[1], 2);
+              out0[2] = amd_bytealign_S (in1[3], in1[2], 2);
+              out0[3] = amd_bytealign_S (     0, in1[3], 2);
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 19:  out0[0] = amd_bytealign_S (in1[1], in1[0], 3);
+              out0[1] = amd_bytealign_S (in1[2], in1[1], 3);
+              out0[2] = amd_bytealign_S (in1[3], in1[2], 3);
+              out0[3] = amd_bytealign_S (     0, in1[3], 3);
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 20:  out0[0] = in1[1];
+              out0[1] = in1[2];
+              out0[2] = in1[3];
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 21:  out0[0] = amd_bytealign_S (in1[2], in1[1], 1);
+              out0[1] = amd_bytealign_S (in1[3], in1[2], 1);
+              out0[2] = amd_bytealign_S (     0, in1[3], 1);
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 22:  out0[0] = amd_bytealign_S (in1[2], in1[1], 2);
+              out0[1] = amd_bytealign_S (in1[3], in1[2], 2);
+              out0[2] = amd_bytealign_S (     0, in1[3], 2);
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 23:  out0[0] = amd_bytealign_S (in1[2], in1[1], 3);
+              out0[1] = amd_bytealign_S (in1[3], in1[2], 3);
+              out0[2] = amd_bytealign_S (     0, in1[3], 3);
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 24:  out0[0] = in1[2];
+              out0[1] = in1[3];
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 25:  out0[0] = amd_bytealign_S (in1[3], in1[2], 1);
+              out0[1] = amd_bytealign_S (     0, in1[3], 1);
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 26:  out0[0] = amd_bytealign_S (in1[3], in1[2], 2);
+              out0[1] = amd_bytealign_S (     0, in1[3], 2);
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 27:  out0[0] = amd_bytealign_S (in1[3], in1[2], 3);
+              out0[1] = amd_bytealign_S (     0, in1[3], 3);
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 28:  out0[0] = in1[3];
+              out0[1] = 0;
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 29:  out0[0] = amd_bytealign_S (     0, in1[3], 1);
+              out0[1] = 0;
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 30:  out0[0] = amd_bytealign_S (     0, in1[3], 2);
+              out0[1] = 0;
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+    case 31:  out0[0] = amd_bytealign_S (     0, in1[3], 3);
+              out0[1] = 0;
+              out0[2] = 0;
+              out0[3] = 0;
+              out1[0] = 0;
+              out1[1] = 0;
+              out1[2] = 0;
+              out1[3] = 0;
+              break;
+  }
+  #endif
+}
+
+inline void rshift_block_N (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4], const u32 num)
+{
+  #ifdef IS_NV
+  switch (num)
+  {
+    case  0:  out1[3] = in1[3];
+              out1[2] = in1[2];
+              out1[1] = in1[1];
+              out1[0] = in1[0];
+              out0[3] = in0[3];
+              out0[2] = in0[2];
+              out0[1] = in0[1];
+              out0[0] = in0[0];
+              break;
+    case  1:  out1[3] = __byte_perm_S (in1[2], in1[3], 0x6543);
+              out1[2] = __byte_perm_S (in1[1], in1[2], 0x6543);
+              out1[1] = __byte_perm_S (in1[0], in1[1], 0x6543);
+              out1[0] = __byte_perm_S (in0[3], in1[0], 0x6543);
+              out0[3] = __byte_perm_S (in0[2], in0[3], 0x6543);
+              out0[2] = __byte_perm_S (in0[1], in0[2], 0x6543);
+              out0[1] = __byte_perm_S (in0[0], in0[1], 0x6543);
+              out0[0] = __byte_perm_S (     0, in0[0], 0x6543);
+              break;
+    case  2:  out1[3] = __byte_perm_S (in1[2], in1[3], 0x5432);
+              out1[2] = __byte_perm_S (in1[1], in1[2], 0x5432);
+              out1[1] = __byte_perm_S (in1[0], in1[1], 0x5432);
+              out1[0] = __byte_perm_S (in0[3], in1[0], 0x5432);
+              out0[3] = __byte_perm_S (in0[2], in0[3], 0x5432);
+              out0[2] = __byte_perm_S (in0[1], in0[2], 0x5432);
+              out0[1] = __byte_perm_S (in0[0], in0[1], 0x5432);
+              out0[0] = __byte_perm_S (     0, in0[0], 0x5432);
+              break;
+    case  3:  out1[3] = __byte_perm_S (in1[2], in1[3], 0x4321);
+              out1[2] = __byte_perm_S (in1[1], in1[2], 0x4321);
+              out1[1] = __byte_perm_S (in1[0], in1[1], 0x4321);
+              out1[0] = __byte_perm_S (in0[3], in1[0], 0x4321);
+              out0[3] = __byte_perm_S (in0[2], in0[3], 0x4321);
+              out0[2] = __byte_perm_S (in0[1], in0[2], 0x4321);
+              out0[1] = __byte_perm_S (in0[0], in0[1], 0x4321);
+              out0[0] = __byte_perm_S (     0, in0[0], 0x4321);
+              break;
+    case  4:  out1[3] = in1[2];
+              out1[2] = in1[1];
+              out1[1] = in1[0];
+              out1[0] = in0[3];
+              out0[3] = in0[2];
+              out0[2] = in0[1];
+              out0[1] = in0[0];
+              out0[0] = 0;
+              break;
+    case  5:  out1[3] = __byte_perm_S (in1[1], in1[2], 0x6543);
+              out1[2] = __byte_perm_S (in1[0], in1[1], 0x6543);
+              out1[1] = __byte_perm_S (in0[3], in1[0], 0x6543);
+              out1[0] = __byte_perm_S (in0[2], in0[3], 0x6543);
+              out0[3] = __byte_perm_S (in0[1], in0[2], 0x6543);
+              out0[2] = __byte_perm_S (in0[0], in0[1], 0x6543);
+              out0[1] = __byte_perm_S (     0, in0[0], 0x6543);
+              out0[0] = 0;
+              break;
+    case  6:  out1[3] = __byte_perm_S (in1[1], in1[2], 0x5432);
+              out1[2] = __byte_perm_S (in1[0], in1[1], 0x5432);
+              out1[1] = __byte_perm_S (in0[3], in1[0], 0x5432);
+              out1[0] = __byte_perm_S (in0[2], in0[3], 0x5432);
+              out0[3] = __byte_perm_S (in0[1], in0[2], 0x5432);
+              out0[2] = __byte_perm_S (in0[0], in0[1], 0x5432);
+              out0[1] = __byte_perm_S (     0, in0[0], 0x5432);
+              out0[0] = 0;
+              break;
+    case  7:  out1[3] = __byte_perm_S (in1[1], in1[2], 0x4321);
+              out1[2] = __byte_perm_S (in1[0], in1[1], 0x4321);
+              out1[1] = __byte_perm_S (in0[3], in1[0], 0x4321);
+              out1[0] = __byte_perm_S (in0[2], in0[3], 0x4321);
+              out0[3] = __byte_perm_S (in0[1], in0[2], 0x4321);
+              out0[2] = __byte_perm_S (in0[0], in0[1], 0x4321);
+              out0[1] = __byte_perm_S (     0, in0[0], 0x4321);
+              out0[0] = 0;
+              break;
+    case  8:  out1[3] = in1[1];
+              out1[2] = in1[0];
+              out1[1] = in0[3];
+              out1[0] = in0[2];
+              out0[3] = in0[1];
+              out0[2] = in0[0];
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case  9:  out1[3] = __byte_perm_S (in1[0], in1[1], 0x6543);
+              out1[2] = __byte_perm_S (in0[3], in1[0], 0x6543);
+              out1[1] = __byte_perm_S (in0[2], in0[3], 0x6543);
+              out1[0] = __byte_perm_S (in0[1], in0[2], 0x6543);
+              out0[3] = __byte_perm_S (in0[0], in0[1], 0x6543);
+              out0[2] = __byte_perm_S (     0, in0[0], 0x6543);
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 10:  out1[3] = __byte_perm_S (in1[0], in1[1], 0x5432);
+              out1[2] = __byte_perm_S (in0[3], in1[0], 0x5432);
+              out1[1] = __byte_perm_S (in0[2], in0[3], 0x5432);
+              out1[0] = __byte_perm_S (in0[1], in0[2], 0x5432);
+              out0[3] = __byte_perm_S (in0[0], in0[1], 0x5432);
+              out0[2] = __byte_perm_S (     0, in0[0], 0x5432);
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 11:  out1[3] = __byte_perm_S (in1[0], in1[1], 0x4321);
+              out1[2] = __byte_perm_S (in0[3], in1[0], 0x4321);
+              out1[1] = __byte_perm_S (in0[2], in0[3], 0x4321);
+              out1[0] = __byte_perm_S (in0[1], in0[2], 0x4321);
+              out0[3] = __byte_perm_S (in0[0], in0[1], 0x4321);
+              out0[2] = __byte_perm_S (     0, in0[0], 0x4321);
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 12:  out1[3] = in1[0];
+              out1[2] = in0[3];
+              out1[1] = in0[2];
+              out1[0] = in0[1];
+              out0[3] = in0[0];
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 13:  out1[3] = __byte_perm_S (in0[3], in1[0], 0x6543);
+              out1[2] = __byte_perm_S (in0[2], in0[3], 0x6543);
+              out1[1] = __byte_perm_S (in0[1], in0[2], 0x6543);
+              out1[0] = __byte_perm_S (in0[0], in0[1], 0x6543);
+              out0[3] = __byte_perm_S (     0, in0[0], 0x6543);
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 14:  out1[3] = __byte_perm_S (in0[3], in1[0], 0x5432);
+              out1[2] = __byte_perm_S (in0[2], in0[3], 0x5432);
+              out1[1] = __byte_perm_S (in0[1], in0[2], 0x5432);
+              out1[0] = __byte_perm_S (in0[0], in0[1], 0x5432);
+              out0[3] = __byte_perm_S (     0, in0[0], 0x5432);
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 15:  out1[3] = __byte_perm_S (in0[3], in1[0], 0x4321);
+              out1[2] = __byte_perm_S (in0[2], in0[3], 0x4321);
+              out1[1] = __byte_perm_S (in0[1], in0[2], 0x4321);
+              out1[0] = __byte_perm_S (in0[0], in0[1], 0x4321);
+              out0[3] = __byte_perm_S (     0, in0[0], 0x4321);
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 16:  out1[3] = in0[3];
+              out1[2] = in0[2];
+              out1[1] = in0[1];
+              out1[0] = in0[0];
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 17:  out1[3] = __byte_perm_S (in0[2], in0[3], 0x6543);
+              out1[2] = __byte_perm_S (in0[1], in0[2], 0x6543);
+              out1[1] = __byte_perm_S (in0[0], in0[1], 0x6543);
+              out1[0] = __byte_perm_S (     0, in0[0], 0x6543);
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 18:  out1[3] = __byte_perm_S (in0[2], in0[3], 0x5432);
+              out1[2] = __byte_perm_S (in0[1], in0[2], 0x5432);
+              out1[1] = __byte_perm_S (in0[0], in0[1], 0x5432);
+              out1[0] = __byte_perm_S (     0, in0[0], 0x5432);
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 19:  out1[3] = __byte_perm_S (in0[2], in0[3], 0x4321);
+              out1[2] = __byte_perm_S (in0[1], in0[2], 0x4321);
+              out1[1] = __byte_perm_S (in0[0], in0[1], 0x4321);
+              out1[0] = __byte_perm_S (     0, in0[0], 0x4321);
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 20:  out1[3] = in0[2];
+              out1[2] = in0[1];
+              out1[1] = in0[0];
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 21:  out1[3] = __byte_perm_S (in0[1], in0[2], 0x6543);
+              out1[2] = __byte_perm_S (in0[0], in0[1], 0x6543);
+              out1[1] = __byte_perm_S (     0, in0[0], 0x6543);
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 22:  out1[3] = __byte_perm_S (in0[1], in0[2], 0x5432);
+              out1[2] = __byte_perm_S (in0[0], in0[1], 0x5432);
+              out1[1] = __byte_perm_S (     0, in0[0], 0x5432);
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 23:  out1[3] = __byte_perm_S (in0[1], in0[2], 0x4321);
+              out1[2] = __byte_perm_S (in0[0], in0[1], 0x4321);
+              out1[1] = __byte_perm_S (     0, in0[0], 0x4321);
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 24:  out1[3] = in0[1];
+              out1[2] = in0[0];
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 25:  out1[3] = __byte_perm_S (in0[0], in0[1], 0x6543);
+              out1[2] = __byte_perm_S (     0, in0[0], 0x6543);
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 26:  out1[3] = __byte_perm_S (in0[0], in0[1], 0x5432);
+              out1[2] = __byte_perm_S (     0, in0[0], 0x5432);
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 27:  out1[3] = __byte_perm_S (in0[0], in0[1], 0x4321);
+              out1[2] = __byte_perm_S (     0, in0[0], 0x4321);
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 28:  out1[3] = in0[0];
+              out1[2] = 0;
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 29:  out1[3] = __byte_perm_S (     0, in0[0], 0x6543);
+              out1[2] = 0;
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 30:  out1[3] = __byte_perm_S (     0, in0[0], 0x5432);
+              out1[2] = 0;
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 31:  out1[3] = __byte_perm_S (     0, in0[0], 0x4321);
+              out1[2] = 0;
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+  }
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  switch (num)
+  {
+    case  0:  out1[3] = in1[3];
+              out1[2] = in1[2];
+              out1[1] = in1[1];
+              out1[0] = in1[0];
+              out0[3] = in0[3];
+              out0[2] = in0[2];
+              out0[1] = in0[1];
+              out0[0] = in0[0];
+              break;
+    case  1:  out1[3] = amd_bytealign_S (in1[3], in1[2], 3);
+              out1[2] = amd_bytealign_S (in1[2], in1[1], 3);
+              out1[1] = amd_bytealign_S (in1[1], in1[0], 3);
+              out1[0] = amd_bytealign_S (in1[0], in0[3], 3);
+              out0[3] = amd_bytealign_S (in0[3], in0[2], 3);
+              out0[2] = amd_bytealign_S (in0[2], in0[1], 3);
+              out0[1] = amd_bytealign_S (in0[1], in0[0], 3);
+              out0[0] = amd_bytealign_S (in0[0],      0, 3);
+              break;
+    case  2:  out1[3] = amd_bytealign_S (in1[3], in1[2], 2);
+              out1[2] = amd_bytealign_S (in1[2], in1[1], 2);
+              out1[1] = amd_bytealign_S (in1[1], in1[0], 2);
+              out1[0] = amd_bytealign_S (in1[0], in0[3], 2);
+              out0[3] = amd_bytealign_S (in0[3], in0[2], 2);
+              out0[2] = amd_bytealign_S (in0[2], in0[1], 2);
+              out0[1] = amd_bytealign_S (in0[1], in0[0], 2);
+              out0[0] = amd_bytealign_S (in0[0],      0, 2);
+              break;
+    case  3:  out1[3] = amd_bytealign_S (in1[3], in1[2], 1);
+              out1[2] = amd_bytealign_S (in1[2], in1[1], 1);
+              out1[1] = amd_bytealign_S (in1[1], in1[0], 1);
+              out1[0] = amd_bytealign_S (in1[0], in0[3], 1);
+              out0[3] = amd_bytealign_S (in0[3], in0[2], 1);
+              out0[2] = amd_bytealign_S (in0[2], in0[1], 1);
+              out0[1] = amd_bytealign_S (in0[1], in0[0], 1);
+              out0[0] = amd_bytealign_S (in0[0],      0, 1);
+              break;
+    case  4:  out1[3] = in1[2];
+              out1[2] = in1[1];
+              out1[1] = in1[0];
+              out1[0] = in0[3];
+              out0[3] = in0[2];
+              out0[2] = in0[1];
+              out0[1] = in0[0];
+              out0[0] = 0;
+              break;
+    case  5:  out1[3] = amd_bytealign_S (in1[2], in1[1], 3);
+              out1[2] = amd_bytealign_S (in1[1], in1[0], 3);
+              out1[1] = amd_bytealign_S (in1[0], in0[3], 3);
+              out1[0] = amd_bytealign_S (in0[3], in0[2], 3);
+              out0[3] = amd_bytealign_S (in0[2], in0[1], 3);
+              out0[2] = amd_bytealign_S (in0[1], in0[0], 3);
+              out0[1] = amd_bytealign_S (in0[0],      0, 3);
+              out0[0] = 0;
+              break;
+    case  6:  out1[3] = amd_bytealign_S (in1[2], in1[1], 2);
+              out1[2] = amd_bytealign_S (in1[1], in1[0], 2);
+              out1[1] = amd_bytealign_S (in1[0], in0[3], 2);
+              out1[0] = amd_bytealign_S (in0[3], in0[2], 2);
+              out0[3] = amd_bytealign_S (in0[2], in0[1], 2);
+              out0[2] = amd_bytealign_S (in0[1], in0[0], 2);
+              out0[1] = amd_bytealign_S (in0[0],      0, 2);
+              out0[0] = 0;
+              break;
+    case  7:  out1[3] = amd_bytealign_S (in1[2], in1[1], 1);
+              out1[2] = amd_bytealign_S (in1[1], in1[0], 1);
+              out1[1] = amd_bytealign_S (in1[0], in0[3], 1);
+              out1[0] = amd_bytealign_S (in0[3], in0[2], 1);
+              out0[3] = amd_bytealign_S (in0[2], in0[1], 1);
+              out0[2] = amd_bytealign_S (in0[1], in0[0], 1);
+              out0[1] = amd_bytealign_S (in0[0],      0, 1);
+              out0[0] = 0;
+              break;
+    case  8:  out1[3] = in1[1];
+              out1[2] = in1[0];
+              out1[1] = in0[3];
+              out1[0] = in0[2];
+              out0[3] = in0[1];
+              out0[2] = in0[0];
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case  9:  out1[3] = amd_bytealign_S (in1[1], in1[0], 3);
+              out1[2] = amd_bytealign_S (in1[0], in0[3], 3);
+              out1[1] = amd_bytealign_S (in0[3], in0[2], 3);
+              out1[0] = amd_bytealign_S (in0[2], in0[1], 3);
+              out0[3] = amd_bytealign_S (in0[1], in0[0], 3);
+              out0[2] = amd_bytealign_S (in0[0],      0, 3);
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 10:  out1[3] = amd_bytealign_S (in1[1], in1[0], 2);
+              out1[2] = amd_bytealign_S (in1[0], in0[3], 2);
+              out1[1] = amd_bytealign_S (in0[3], in0[2], 2);
+              out1[0] = amd_bytealign_S (in0[2], in0[1], 2);
+              out0[3] = amd_bytealign_S (in0[1], in0[0], 2);
+              out0[2] = amd_bytealign_S (in0[0],      0, 2);
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 11:  out1[3] = amd_bytealign_S (in1[1], in1[0], 1);
+              out1[2] = amd_bytealign_S (in1[0], in0[3], 1);
+              out1[1] = amd_bytealign_S (in0[3], in0[2], 1);
+              out1[0] = amd_bytealign_S (in0[2], in0[1], 1);
+              out0[3] = amd_bytealign_S (in0[1], in0[0], 1);
+              out0[2] = amd_bytealign_S (in0[0],      0, 1);
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 12:  out1[3] = in1[0];
+              out1[2] = in0[3];
+              out1[1] = in0[2];
+              out1[0] = in0[1];
+              out0[3] = in0[0];
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 13:  out1[3] = amd_bytealign_S (in1[0], in0[3], 3);
+              out1[2] = amd_bytealign_S (in0[3], in0[2], 3);
+              out1[1] = amd_bytealign_S (in0[2], in0[1], 3);
+              out1[0] = amd_bytealign_S (in0[1], in0[0], 3);
+              out0[3] = amd_bytealign_S (in0[0],      0, 3);
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 14:  out1[3] = amd_bytealign_S (in1[0], in0[3], 2);
+              out1[2] = amd_bytealign_S (in0[3], in0[2], 2);
+              out1[1] = amd_bytealign_S (in0[2], in0[1], 2);
+              out1[0] = amd_bytealign_S (in0[1], in0[0], 2);
+              out0[3] = amd_bytealign_S (in0[0],      0, 2);
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 15:  out1[3] = amd_bytealign_S (in1[0], in0[3], 1);
+              out1[2] = amd_bytealign_S (in0[3], in0[2], 1);
+              out1[1] = amd_bytealign_S (in0[2], in0[1], 1);
+              out1[0] = amd_bytealign_S (in0[1], in0[0], 1);
+              out0[3] = amd_bytealign_S (in0[0],      0, 1);
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 16:  out1[3] = in0[3];
+              out1[2] = in0[2];
+              out1[1] = in0[1];
+              out1[0] = in0[0];
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 17:  out1[3] = amd_bytealign_S (in0[3], in0[2], 3);
+              out1[2] = amd_bytealign_S (in0[2], in0[1], 3);
+              out1[1] = amd_bytealign_S (in0[1], in0[0], 3);
+              out1[0] = amd_bytealign_S (in0[0],      0, 3);
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 18:  out1[3] = amd_bytealign_S (in0[3], in0[2], 2);
+              out1[2] = amd_bytealign_S (in0[2], in0[1], 2);
+              out1[1] = amd_bytealign_S (in0[1], in0[0], 2);
+              out1[0] = amd_bytealign_S (in0[0],      0, 2);
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 19:  out1[3] = amd_bytealign_S (in0[3], in0[2], 1);
+              out1[2] = amd_bytealign_S (in0[2], in0[1], 1);
+              out1[1] = amd_bytealign_S (in0[1], in0[0], 1);
+              out1[0] = amd_bytealign_S (in0[0],      0, 1);
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 20:  out1[3] = in0[2];
+              out1[2] = in0[1];
+              out1[1] = in0[0];
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 21:  out1[3] = amd_bytealign_S (in0[2], in0[1], 3);
+              out1[2] = amd_bytealign_S (in0[1], in0[0], 3);
+              out1[1] = amd_bytealign_S (in0[0],      0, 3);
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 22:  out1[3] = amd_bytealign_S (in0[2], in0[1], 2);
+              out1[2] = amd_bytealign_S (in0[1], in0[0], 2);
+              out1[1] = amd_bytealign_S (in0[0],      0, 2);
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 23:  out1[3] = amd_bytealign_S (in0[2], in0[1], 1);
+              out1[2] = amd_bytealign_S (in0[1], in0[0], 1);
+              out1[1] = amd_bytealign_S (in0[0],      0, 1);
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 24:  out1[3] = in0[1];
+              out1[2] = in0[0];
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 25:  out1[3] = amd_bytealign_S (in0[1], in0[0], 3);
+              out1[2] = amd_bytealign_S (in0[0],      0, 3);
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 26:  out1[3] = amd_bytealign_S (in0[1], in0[0], 2);
+              out1[2] = amd_bytealign_S (in0[0],      0, 2);
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 27:  out1[3] = amd_bytealign_S (in0[1], in0[0], 1);
+              out1[2] = amd_bytealign_S (in0[0],      0, 1);
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 28:  out1[3] = in0[0];
+              out1[2] = 0;
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 29:  out1[3] = amd_bytealign_S (in0[0],      0, 3);
+              out1[2] = 0;
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 30:  out1[3] = amd_bytealign_S (in0[0],      0, 2);
+              out1[2] = 0;
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+    case 31:  out1[3] = amd_bytealign_S (in0[0],      0, 1);
+              out1[2] = 0;
+              out1[1] = 0;
+              out1[0] = 0;
+              out0[3] = 0;
+              out0[2] = 0;
+              out0[1] = 0;
+              out0[0] = 0;
+              break;
+  }
+  #endif
+}
+
+inline void append_block1 (const u32 offset, u32 dst0[4], u32 dst1[4], const u32 src_r0)
+{
+  u32 tmp[2];
+
+  switch (offset & 3)
+  {
+    case  0:  tmp[0] = src_r0;
+              tmp[1] = 0;
+              break;
+    case  1:  tmp[0] = src_r0 <<  8;
+              tmp[1] = src_r0 >> 24;
+              break;
+    case  2:  tmp[0] = src_r0 << 16;
+              tmp[1] = src_r0 >> 16;
+              break;
+    case  3:  tmp[0] = src_r0 << 24;
+              tmp[1] = src_r0 >>  8;
+              break;
+  }
+
+  switch (offset / 4)
+  {
+    case  0:  dst0[0] |= tmp[0];
+              dst0[1]  = tmp[1];
+              break;
+    case  1:  dst0[1] |= tmp[0];
+              dst0[2]  = tmp[1];
+              break;
+    case  2:  dst0[2] |= tmp[0];
+              dst0[3]  = tmp[1];
+              break;
+    case  3:  dst0[3] |= tmp[0];
+              dst1[0]  = tmp[1];
+              break;
+    case  4:  dst1[0] |= tmp[0];
+              dst1[1]  = tmp[1];
+              break;
+    case  5:  dst1[1] |= tmp[0];
+              dst1[2]  = tmp[1];
+              break;
+    case  6:  dst1[2] |= tmp[0];
+              dst1[3]  = tmp[1];
+              break;
+    case  7:  dst1[3] |= tmp[0];
+              break;
+  }
+}
+
+inline void append_block8 (const u32 offset, u32 dst0[4], u32 dst1[4], const u32 src_l0[4], const u32 src_l1[4], const u32 src_r0[4], const u32 src_r1[4])
+{
+/*
+  #ifdef IS_NV
+  switch (offset)
+  {
+    case 0:
+      dst0[0] = src_r0[0];
+      dst0[1] = src_r0[1];
+      dst0[2] = src_r0[2];
+      dst0[3] = src_r0[3];
+      dst1[0] = src_r1[0];
+      dst1[1] = src_r1[1];
+      dst1[2] = src_r1[2];
+      dst1[3] = src_r1[3];
+      break;
+
+    case 1:
+      dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x6540);
+      dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
+      dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
+      dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
+      dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543);
+      dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543);
+      dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x6543);
+      dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x6543);
+      break;
+
+    case 2:
+      dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x5410);
+      dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
+      dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
+      dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
+      dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432);
+      dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432);
+      dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x5432);
+      dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x5432);
+      break;
+
+    case 3:
+      dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x4210);
+      dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
+      dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
+      dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
+      dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321);
+      dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321);
+      dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x4321);
+      dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x4321);
+      break;
+
+    case 4:
+      dst0[1] = src_r0[0];
+      dst0[2] = src_r0[1];
+      dst0[3] = src_r0[2];
+      dst1[0] = src_r0[3];
+      dst1[1] = src_r1[0];
+      dst1[2] = src_r1[1];
+      dst1[3] = src_r1[2];
+      break;
+
+    case 5:
+      dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x6540);
+      dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
+      dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
+      dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
+      dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543);
+      dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543);
+      dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x6543);
+      break;
+
+    case 6:
+      dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x5410);
+      dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
+      dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
+      dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
+      dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432);
+      dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432);
+      dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x5432);
+      break;
+
+    case 7:
+      dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x4210);
+      dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
+      dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
+      dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
+      dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321);
+      dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321);
+      dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x4321);
+      break;
+
+    case 8:
+      dst0[2] = src_r0[0];
+      dst0[3] = src_r0[1];
+      dst1[0] = src_r0[2];
+      dst1[1] = src_r0[3];
+      dst1[2] = src_r1[0];
+      dst1[3] = src_r1[1];
+      break;
+
+    case 9:
+      dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x6540);
+      dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
+      dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
+      dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
+      dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543);
+      dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543);
+      break;
+
+    case 10:
+      dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x5410);
+      dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
+      dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
+      dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
+      dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432);
+      dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432);
+      break;
+
+    case 11:
+      dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x4210);
+      dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
+      dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
+      dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
+      dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321);
+      dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321);
+      break;
+
+    case 12:
+      dst0[3] = src_r0[0];
+      dst1[0] = src_r0[1];
+      dst1[1] = src_r0[2];
+      dst1[2] = src_r0[3];
+      dst1[3] = src_r1[0];
+      break;
+
+    case 13:
+      dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x6540);
+      dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
+      dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
+      dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
+      dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543);
+      break;
+
+    case 14:
+      dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x5410);
+      dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
+      dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
+      dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
+      dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432);
+      break;
+
+    case 15:
+      dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x4210);
+      dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
+      dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
+      dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
+      dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321);
+      break;
+
+    case 16:
+      dst1[0] = src_r0[0];
+      dst1[1] = src_r0[1];
+      dst1[2] = src_r0[2];
+      dst1[3] = src_r0[3];
+      break;
+
+    case 17:
+      dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x6540);
+      dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
+      dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
+      dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
+      break;
+
+    case 18:
+      dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x5410);
+      dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
+      dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
+      dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
+      break;
+
+    case 19:
+      dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x4210);
+      dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
+      dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
+      dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
+      break;
+
+    case 20:
+      dst1[1] = src_r0[0];
+      dst1[2] = src_r0[1];
+      dst1[3] = src_r0[2];
+      break;
+
+    case 21:
+      dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x6540);
+      dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
+      dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
+      break;
+
+    case 22:
+      dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x5410);
+      dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
+      dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
+      break;
+
+    case 23:
+      dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x4210);
+      dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
+      dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
+      break;
+
+    case 24:
+      dst1[2] = src_r0[0];
+      dst1[3] = src_r0[1];
+      break;
+
+    case 25:
+      dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x6540);
+      dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
+      break;
+
+    case 26:
+      dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x5410);
+      dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
+      break;
+
+    case 27:
+      dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x4210);
+      dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
+      break;
+
+    case 28:
+      dst1[3] = src_r0[0];
+      break;
+
+    case 29:
+      dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x6540);
+      break;
+
+    case 30:
+      dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x5410);
+      break;
+
+    case 31:
+      dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x4210);
+      break;
+  }
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+*/
+  switch (offset)
+  {
+    case 31:
+      dst1[3] = src_l1[3] | src_r0[0] << 24;
+      break;
+    case 30:
+      dst1[3] = src_l1[3] | src_r0[0] << 16;
+      break;
+    case 29:
+      dst1[3] = src_l1[3] | src_r0[0] <<  8;
+      break;
+    case 28:
+      dst1[3] = src_r0[0];
+      break;
+    case 27:
+      dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
+      dst1[2] = src_l1[2] | src_r0[0] << 24;
+      break;
+    case 26:
+      dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
+      dst1[2] = src_l1[2] | src_r0[0] << 16;
+      break;
+    case 25:
+      dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
+      dst1[2] = src_l1[2] | src_r0[0] <<  8;
+      break;
+    case 24:
+      dst1[3] = src_r0[1];
+      dst1[2] = src_r0[0];
+      break;
+    case 23:
+      dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
+      dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
+      dst1[1] = src_l1[1] | src_r0[0] << 24;
+      break;
+    case 22:
+      dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
+      dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
+      dst1[1] = src_l1[1] | src_r0[0] << 16;
+      break;
+    case 21:
+      dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
+      dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
+      dst1[1] = src_l1[1] | src_r0[0] <<  8;
+      break;
+    case 20:
+      dst1[3] = src_r0[2];
+      dst1[2] = src_r0[1];
+      dst1[1] = src_r0[0];
+      break;
+    case 19:
+      dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
+      dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
+      dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
+      dst1[0] = src_l1[0] | src_r0[0] << 24;
+      break;
+    case 18:
+      dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
+      dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
+      dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
+      dst1[0] = src_l1[0] | src_r0[0] << 16;
+      break;
+    case 17:
+      dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
+      dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
+      dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
+      dst1[0] = src_l1[0] | src_r0[0] <<  8;
+      break;
+    case 16:
+      dst1[3] = src_r0[3];
+      dst1[2] = src_r0[2];
+      dst1[1] = src_r0[1];
+      dst1[0] = src_r0[0];
+      break;
+    case 15:
+      dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 1);
+      dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
+      dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
+      dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
+      dst0[3] = src_l0[3] | src_r0[0] << 24;
+      break;
+    case 14:
+      dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 2);
+      dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
+      dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
+      dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
+      dst0[3] = src_l0[3] | src_r0[0] << 16;
+      break;
+    case 13:
+      dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 3);
+      dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
+      dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
+      dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
+      dst0[3] = src_l0[3] | src_r0[0] <<  8;
+      break;
+    case 12:
+      dst1[3] = src_r1[0];
+      dst1[2] = src_r0[3];
+      dst1[1] = src_r0[2];
+      dst1[0] = src_r0[1];
+      dst0[3] = src_r0[0];
+      break;
+    case 11:
+      dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 1);
+      dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 1);
+      dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
+      dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
+      dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
+      dst0[2] = src_l0[2] | src_r0[0] << 24;
+      break;
+    case 10:
+      dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 2);
+      dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 2);
+      dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
+      dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
+      dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
+      dst0[2] = src_l0[2] | src_r0[0] << 16;
+      break;
+    case 9:
+      dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 3);
+      dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 3);
+      dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
+      dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
+      dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
+      dst0[2] = src_l0[2] | src_r0[0] <<  8;
+      break;
+    case 8:
+      dst1[3] = src_r1[1];
+      dst1[2] = src_r1[0];
+      dst1[1] = src_r0[3];
+      dst1[0] = src_r0[2];
+      dst0[3] = src_r0[1];
+      dst0[2] = src_r0[0];
+      break;
+    case 7:
+      dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 1);
+      dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 1);
+      dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 1);
+      dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
+      dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
+      dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
+      dst0[1] = src_l0[1] | src_r0[0] << 24;
+      break;
+    case 6:
+      dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 2);
+      dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 2);
+      dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 2);
+      dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
+      dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
+      dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
+      dst0[1] = src_l0[1] | src_r0[0] << 16;
+      break;
+    case 5:
+      dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 3);
+      dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 3);
+      dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 3);
+      dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
+      dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
+      dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
+      dst0[1] = src_l0[1] | src_r0[0] <<  8;
+      break;
+    case 4:
+      dst1[3] = src_r1[2];
+      dst1[2] = src_r1[1];
+      dst1[1] = src_r1[0];
+      dst1[0] = src_r0[3];
+      dst0[3] = src_r0[2];
+      dst0[2] = src_r0[1];
+      dst0[1] = src_r0[0];
+      break;
+    case 3:
+      dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 1);
+      dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 1);
+      dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 1);
+      dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 1);
+      dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
+      dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
+      dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
+      dst0[0] = src_l0[0] | src_r0[0] << 24;
+      break;
+    case 2:
+      dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 2);
+      dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 2);
+      dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 2);
+      dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 2);
+      dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
+      dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
+      dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
+      dst0[0] = src_l0[0] | src_r0[0] << 16;
+      break;
+    case 1:
+      dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 3);
+      dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 3);
+      dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 3);
+      dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 3);
+      dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
+      dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
+      dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
+      dst0[0] = src_l0[0] | src_r0[0] <<  8;
+      break;
+    case 0:
+      dst1[3] = src_r1[3];
+      dst1[2] = src_r1[2];
+      dst1[1] = src_r1[1];
+      dst1[0] = src_r1[0];
+      dst0[3] = src_r0[3];
+      dst0[2] = src_r0[2];
+      dst0[1] = src_r0[1];
+      dst0[0] = src_r0[0];
+      break;
+  }
+//  #endif
+}
+
+inline void reverse_block (u32 in0[4], u32 in1[4], u32 out0[4], u32 out1[4], const u32 len)
+{
+  rshift_block_N (in0, in1, out0, out1, 32 - len);
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  tib40[0] = out1[3];
+  tib40[1] = out1[2];
+  tib40[2] = out1[1];
+  tib40[3] = out1[0];
+  tib41[0] = out0[3];
+  tib41[1] = out0[2];
+  tib41[2] = out0[1];
+  tib41[3] = out0[0];
+
+  out0[0] = swap32_S (tib40[0]);
+  out0[1] = swap32_S (tib40[1]);
+  out0[2] = swap32_S (tib40[2]);
+  out0[3] = swap32_S (tib40[3]);
+  out1[0] = swap32_S (tib41[0]);
+  out1[1] = swap32_S (tib41[1]);
+  out1[2] = swap32_S (tib41[2]);
+  out1[3] = swap32_S (tib41[3]);
+}
+
+inline u32 rule_op_mangle_lrest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  buf0[0] |= (generate_cmask (buf0[0]));
+  buf0[1] |= (generate_cmask (buf0[1]));
+  buf0[2] |= (generate_cmask (buf0[2]));
+  buf0[3] |= (generate_cmask (buf0[3]));
+  buf1[0] |= (generate_cmask (buf1[0]));
+  buf1[1] |= (generate_cmask (buf1[1]));
+  buf1[2] |= (generate_cmask (buf1[2]));
+  buf1[3] |= (generate_cmask (buf1[3]));
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_urest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  buf0[0] &= ~(generate_cmask (buf0[0]));
+  buf0[1] &= ~(generate_cmask (buf0[1]));
+  buf0[2] &= ~(generate_cmask (buf0[2]));
+  buf0[3] &= ~(generate_cmask (buf0[3]));
+  buf1[0] &= ~(generate_cmask (buf1[0]));
+  buf1[1] &= ~(generate_cmask (buf1[1]));
+  buf1[2] &= ~(generate_cmask (buf1[2]));
+  buf1[3] &= ~(generate_cmask (buf1[3]));
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_lrest_ufirst (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  rule_op_mangle_lrest (p0, p1, buf0, buf1, in_len);
+
+  buf0[0] &= ~(0x00000020 & generate_cmask (buf0[0]));
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_urest_lfirst (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  rule_op_mangle_urest (p0, p1, buf0, buf1, in_len);
+
+  buf0[0] |= (0x00000020 & generate_cmask (buf0[0]));
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_trest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  buf0[0] ^= (generate_cmask (buf0[0]));
+  buf0[1] ^= (generate_cmask (buf0[1]));
+  buf0[2] ^= (generate_cmask (buf0[2]));
+  buf0[3] ^= (generate_cmask (buf0[3]));
+  buf1[0] ^= (generate_cmask (buf1[0]));
+  buf1[1] ^= (generate_cmask (buf1[1]));
+  buf1[2] ^= (generate_cmask (buf1[2]));
+  buf1[3] ^= (generate_cmask (buf1[3]));
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_toggle_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  const u32 tmp = 0x20u << ((p0 & 3) * 8);
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] ^= (tmp & generate_cmask (buf0[0])); break;
+    case  1:  buf0[1] ^= (tmp & generate_cmask (buf0[1])); break;
+    case  2:  buf0[2] ^= (tmp & generate_cmask (buf0[2])); break;
+    case  3:  buf0[3] ^= (tmp & generate_cmask (buf0[3])); break;
+    case  4:  buf1[0] ^= (tmp & generate_cmask (buf1[0])); break;
+    case  5:  buf1[1] ^= (tmp & generate_cmask (buf1[1])); break;
+    case  6:  buf1[2] ^= (tmp & generate_cmask (buf1[2])); break;
+    case  7:  buf1[3] ^= (tmp & generate_cmask (buf1[3])); break;
+  }
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_reverse (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  reverse_block (buf0, buf1, buf0, buf1, in_len);
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_dupeword (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if ((in_len + in_len) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  append_block8 (out_len, buf0, buf1, buf0, buf1, buf0, buf1);
+
+  out_len += in_len;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_dupeword_times (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (((in_len * p0) + in_len) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  tib40[0] = buf0[0];
+  tib40[1] = buf0[1];
+  tib40[2] = buf0[2];
+  tib40[3] = buf0[3];
+  tib41[0] = buf1[0];
+  tib41[1] = buf1[1];
+  tib41[2] = buf1[2];
+  tib41[3] = buf1[3];
+
+  for (u32 i = 0; i < p0; i++)
+  {
+    append_block8 (out_len, buf0, buf1, buf0, buf1, tib40, tib41);
+
+    out_len += in_len;
+  }
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_reflect (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if ((in_len + in_len) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  reverse_block (buf0, buf1, tib40, tib41, out_len);
+
+  append_block8 (out_len, buf0, buf1, buf0, buf1, tib40, tib41);
+
+  out_len += in_len;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_append (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if ((in_len + 1) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  append_block1 (out_len, buf0, buf1, p0);
+
+  out_len++;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_prepend (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if ((in_len + 1) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  rshift_block (buf0, buf1, buf0, buf1);
+
+  buf0[0] = buf0[0] | p0;
+
+  out_len++;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_rotate_left (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (in_len == 0) return (in_len);
+
+  const u32 in_len1 = in_len - 1;
+
+  const u32 sh = (in_len1 & 3) * 8;
+
+  const u32 tmp = (buf0[0] & 0xff) << sh;
+
+  lshift_block (buf0, buf1, buf0, buf1);
+
+  switch (in_len1 / 4)
+  {
+    case  0:  buf0[0] |= tmp; break;
+    case  1:  buf0[1] |= tmp; break;
+    case  2:  buf0[2] |= tmp; break;
+    case  3:  buf0[3] |= tmp; break;
+    case  4:  buf1[0] |= tmp; break;
+    case  5:  buf1[1] |= tmp; break;
+    case  6:  buf1[2] |= tmp; break;
+    case  7:  buf1[3] |= tmp; break;
+  }
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_rotate_right (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (in_len == 0) return (in_len);
+
+  const u32 in_len1 = in_len - 1;
+
+  const u32 sh = (in_len1 & 3) * 8;
+
+  u32 tmp = 0;
+
+  switch (in_len1 / 4)
+  {
+    case  0:  tmp = (buf0[0] >> sh) & 0xff; break;
+    case  1:  tmp = (buf0[1] >> sh) & 0xff; break;
+    case  2:  tmp = (buf0[2] >> sh) & 0xff; break;
+    case  3:  tmp = (buf0[3] >> sh) & 0xff; break;
+    case  4:  tmp = (buf1[0] >> sh) & 0xff; break;
+    case  5:  tmp = (buf1[1] >> sh) & 0xff; break;
+    case  6:  tmp = (buf1[2] >> sh) & 0xff; break;
+    case  7:  tmp = (buf1[3] >> sh) & 0xff; break;
+  }
+
+  rshift_block (buf0, buf1, buf0, buf1);
+
+  buf0[0] |= tmp;
+
+  truncate_right (buf0, buf1, in_len);
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_delete_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (in_len == 0) return (in_len);
+
+  const u32 in_len1 = in_len - 1;
+
+  lshift_block (buf0, buf1, buf0, buf1);
+
+  return in_len1;
+}
+
+inline u32 rule_op_mangle_delete_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (in_len == 0) return (in_len);
+
+  const u32 in_len1 = in_len - 1;
+
+  const u32 tmp = (1 << ((in_len1 & 3) * 8)) - 1;
+
+  switch (in_len1 / 4)
+  {
+    case  0:  buf0[0] &= tmp; break;
+    case  1:  buf0[1] &= tmp; break;
+    case  2:  buf0[2] &= tmp; break;
+    case  3:  buf0[3] &= tmp; break;
+    case  4:  buf1[0] &= tmp; break;
+    case  5:  buf1[1] &= tmp; break;
+    case  6:  buf1[2] &= tmp; break;
+    case  7:  buf1[3] &= tmp; break;
+  }
+
+  return in_len1;
+}
+
+inline u32 rule_op_mangle_delete_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  u32 out_len = in_len;
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  lshift_block (buf0, buf1, tib40, tib41);
+
+  const u32 ml = (1 << ((p0 & 3) * 8)) - 1;
+  const u32 mr = ~ml;
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] =  (buf0[0] & ml)
+                      | (tib40[0] & mr);
+              buf0[1] =  tib40[1];
+              buf0[2] =  tib40[2];
+              buf0[3] =  tib40[3];
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  1:  buf0[1] =  (buf0[1] & ml)
+                      | (tib40[1] & mr);
+              buf0[2] =  tib40[2];
+              buf0[3] =  tib40[3];
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  2:  buf0[2] =  (buf0[2] & ml)
+                      | (tib40[2] & mr);
+              buf0[3] =  tib40[3];
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  3:  buf0[3] =  (buf0[3] & ml)
+                      | (tib40[3] & mr);
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  4:  buf1[0] =  (buf1[0] & ml)
+                      | (tib41[0] & mr);
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  5:  buf1[1] =  (buf1[1] & ml)
+                      | (tib41[1] & mr);
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  6:  buf1[2] =  (buf1[2] & ml)
+                      | (tib41[2] & mr);
+              buf1[3] =  tib41[3];
+              break;
+    case  7:  buf1[3] =  (buf1[3] & ml)
+                      | (tib41[3] & mr);
+              break;
+  }
+
+  out_len--;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_extract (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  if ((p0 + p1) > in_len) return (in_len);
+
+  u32 out_len = p1;
+
+  lshift_block_N (buf0, buf1, buf0, buf1, p0);
+
+  truncate_right (buf0, buf1, out_len);
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_omit (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  if ((p0 + p1) > in_len) return (in_len);
+
+  u32 out_len = in_len;
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  tib40[0] = 0;
+  tib40[1] = 0;
+  tib40[2] = 0;
+  tib40[3] = 0;
+  tib41[0] = 0;
+  tib41[1] = 0;
+  tib41[2] = 0;
+  tib41[3] = 0;
+
+  lshift_block_N (buf0, buf1, tib40, tib41, p1);
+
+  const u32 ml = (1 << ((p0 & 3) * 8)) - 1;
+  const u32 mr = ~ml;
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] =  (buf0[0] & ml)
+                      | (tib40[0] & mr);
+              buf0[1] =  tib40[1];
+              buf0[2] =  tib40[2];
+              buf0[3] =  tib40[3];
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  1:  buf0[1] =  (buf0[1] & ml)
+                      | (tib40[1] & mr);
+              buf0[2] =  tib40[2];
+              buf0[3] =  tib40[3];
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  2:  buf0[2] =  (buf0[2] & ml)
+                      | (tib40[2] & mr);
+              buf0[3] =  tib40[3];
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  3:  buf0[3] =  (buf0[3] & ml)
+                      | (tib40[3] & mr);
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  4:  buf1[0] =  (buf1[0] & ml)
+                      | (tib41[0] & mr);
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  5:  buf1[1] =  (buf1[1] & ml)
+                      | (tib41[1] & mr);
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  6:  buf1[2] =  (buf1[2] & ml)
+                      | (tib41[2] & mr);
+              buf1[3] =  tib41[3];
+              break;
+    case  7:  buf1[3] =  (buf1[3] & ml)
+                      | (tib41[3] & mr);
+              break;
+  }
+
+  out_len -= p1;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_insert (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 > in_len) return (in_len);
+
+  if ((in_len + 1) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  rshift_block (buf0, buf1, tib40, tib41);
+
+  const u32 p1n = p1 << ((p0 & 3) * 8);
+
+  const u32 ml = (1 << ((p0 & 3) * 8)) - 1;
+
+  const u32 mr = 0xffffff00 << ((p0 & 3) * 8);
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] =  (buf0[0] & ml) | p1n | (tib40[0] & mr);
+              buf0[1] =  tib40[1];
+              buf0[2] =  tib40[2];
+              buf0[3] =  tib40[3];
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  1:  buf0[1] =  (buf0[1] & ml) | p1n | (tib40[1] & mr);
+              buf0[2] =  tib40[2];
+              buf0[3] =  tib40[3];
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  2:  buf0[2] =  (buf0[2] & ml) | p1n | (tib40[2] & mr);
+              buf0[3] =  tib40[3];
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  3:  buf0[3] =  (buf0[3] & ml) | p1n | (tib40[3] & mr);
+              buf1[0] =  tib41[0];
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  4:  buf1[0] =  (buf1[0] & ml) | p1n | (tib41[0] & mr);
+              buf1[1] =  tib41[1];
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  5:  buf1[1] =  (buf1[1] & ml) | p1n | (tib41[1] & mr);
+              buf1[2] =  tib41[2];
+              buf1[3] =  tib41[3];
+              break;
+    case  6:  buf1[2] =  (buf1[2] & ml) | p1n | (tib41[2] & mr);
+              buf1[3] =  tib41[3];
+              break;
+    case  7:  buf1[3] =  (buf1[3] & ml) | p1n | (tib41[3] & mr);
+              break;
+  }
+
+  out_len++;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_overstrike (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  const u32 p1n = p1 << ((p0 & 3) * 8);
+
+  const u32 m = ~(0xffu << ((p0 & 3) * 8));
+
+  switch (p0 / 4)
+  {
+    case  0: buf0[0] = (buf0[0] & m) | p1n; break;
+    case  1: buf0[1] = (buf0[1] & m) | p1n; break;
+    case  2: buf0[2] = (buf0[2] & m) | p1n; break;
+    case  3: buf0[3] = (buf0[3] & m) | p1n; break;
+    case  4: buf1[0] = (buf1[0] & m) | p1n; break;
+    case  5: buf1[1] = (buf1[1] & m) | p1n; break;
+    case  6: buf1[2] = (buf1[2] & m) | p1n; break;
+    case  7: buf1[3] = (buf1[3] & m) | p1n; break;
+  }
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_truncate_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  truncate_right (buf0, buf1, p0);
+
+  return p0;
+}
+
+inline u32 rule_op_mangle_replace (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  #ifdef IS_NV
+  for (u32 i = 0; i < in_len; i++)
+  {
+    switch (i)
+    {
+      case  0:  if ((__byte_perm_S (buf0[0], 0, 0x6540)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7650);
+                break;
+      case  1:  if ((__byte_perm_S (buf0[0], 0, 0x6541)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7604);
+                break;
+      case  2:  if ((__byte_perm_S (buf0[0], 0, 0x6542)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7054);
+                break;
+      case  3:  if ((__byte_perm_S (buf0[0], 0, 0x6543)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x0654);
+                break;
+      case  4:  if ((__byte_perm_S (buf0[1], 0, 0x6540)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7650);
+                break;
+      case  5:  if ((__byte_perm_S (buf0[1], 0, 0x6541)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7604);
+                break;
+      case  6:  if ((__byte_perm_S (buf0[1], 0, 0x6542)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7054);
+                break;
+      case  7:  if ((__byte_perm_S (buf0[1], 0, 0x6543)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x0654);
+                break;
+      case  8:  if ((__byte_perm_S (buf0[2], 0, 0x6540)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7650);
+                break;
+      case  9:  if ((__byte_perm_S (buf0[2], 0, 0x6541)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7604);
+                break;
+      case 10:  if ((__byte_perm_S (buf0[2], 0, 0x6542)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7054);
+                break;
+      case 11:  if ((__byte_perm_S (buf0[2], 0, 0x6543)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x0654);
+                break;
+      case 12:  if ((__byte_perm_S (buf0[3], 0, 0x6540)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7650);
+                break;
+      case 13:  if ((__byte_perm_S (buf0[3], 0, 0x6541)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7604);
+                break;
+      case 14:  if ((__byte_perm_S (buf0[3], 0, 0x6542)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7054);
+                break;
+      case 15:  if ((__byte_perm_S (buf0[3], 0, 0x6543)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x0654);
+                break;
+      case 16:  if ((__byte_perm_S (buf1[0], 0, 0x6540)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7650);
+                break;
+      case 17:  if ((__byte_perm_S (buf1[0], 0, 0x6541)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7604);
+                break;
+      case 18:  if ((__byte_perm_S (buf1[0], 0, 0x6542)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7054);
+                break;
+      case 19:  if ((__byte_perm_S (buf1[0], 0, 0x6543)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x0654);
+                break;
+      case 20:  if ((__byte_perm_S (buf1[1], 0, 0x6540)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7650);
+                break;
+      case 21:  if ((__byte_perm_S (buf1[1], 0, 0x6541)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7604);
+                break;
+      case 22:  if ((__byte_perm_S (buf1[1], 0, 0x6542)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7054);
+                break;
+      case 23:  if ((__byte_perm_S (buf1[1], 0, 0x6543)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x0654);
+                break;
+      case 24:  if ((__byte_perm_S (buf1[2], 0, 0x6540)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7650);
+                break;
+      case 25:  if ((__byte_perm_S (buf1[2], 0, 0x6541)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7604);
+                break;
+      case 26:  if ((__byte_perm_S (buf1[2], 0, 0x6542)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7054);
+                break;
+      case 27:  if ((__byte_perm_S (buf1[2], 0, 0x6543)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x0654);
+                break;
+      case 28:  if ((__byte_perm_S (buf1[3], 0, 0x6540)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7650);
+                break;
+      case 29:  if ((__byte_perm_S (buf1[3], 0, 0x6541)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7604);
+                break;
+      case 30:  if ((__byte_perm_S (buf1[3], 0, 0x6542)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7054);
+                break;
+      case 31:  if ((__byte_perm_S (buf1[3], 0, 0x6543)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x0654);
+                break;
+    }
+  }
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  const uchar4 tmp0 = (uchar4) (p0);
+  const uchar4 tmp1 = (uchar4) (p1);
+
+  uchar4 tmp;
+
+  tmp = as_uchar4 (buf0[0]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[0] = as_uint (tmp);
+  tmp = as_uchar4 (buf0[1]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[1] = as_uint (tmp);
+  tmp = as_uchar4 (buf0[2]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[2] = as_uint (tmp);
+  tmp = as_uchar4 (buf0[3]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[3] = as_uint (tmp);
+  tmp = as_uchar4 (buf1[0]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[0] = as_uint (tmp);
+  tmp = as_uchar4 (buf1[1]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[1] = as_uint (tmp);
+  tmp = as_uchar4 (buf1[2]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[2] = as_uint (tmp);
+  tmp = as_uchar4 (buf1[3]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[3] = as_uint (tmp);
+  #endif
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_purgechar (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  // TODO
+  return in_len;
+}
+
+inline u32 rule_op_mangle_togglecase_rec (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  // TODO
+  return in_len;
+}
+
+inline u32 rule_op_mangle_dupechar_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if ( in_len       ==  0) return (in_len);
+  if ((in_len + p0) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  const u32 tmp = buf0[0] & 0xFF;
+
+  rshift_block_N (buf0, buf1, buf0, buf1, p0);
+
+  #ifdef IS_NV
+  switch (p0)
+  {
+    case  1:  buf0[0] |= tmp;
+              break;
+    case  2:  buf0[0] |= __byte_perm_S (tmp, 0, 0x5400);
+              break;
+    case  3:  buf0[0] |= __byte_perm_S (tmp, 0, 0x4000);
+              break;
+    case  4:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              break;
+    case  5:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= tmp;
+              break;
+    case  6:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x5400);
+              break;
+    case  7:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x4000);
+              break;
+    case  8:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              break;
+    case  9:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= tmp;
+              break;
+    case 10:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x5400);
+              break;
+    case 11:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x4000);
+              break;
+    case 12:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              break;
+    case 13:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= tmp;
+              break;
+    case 14:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x5400);
+              break;
+    case 15:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x4000);
+              break;
+    case 16:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              break;
+    case 17:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= tmp;
+              break;
+    case 18:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x5400);
+              break;
+    case 19:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x4000);
+              break;
+    case 20:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              break;
+    case 21:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= tmp;
+              break;
+    case 22:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x5400);
+              break;
+    case 23:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x4000);
+              break;
+    case 24:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              break;
+    case 25:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[2] |= tmp;
+              break;
+    case 26:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[2] |= __byte_perm_S (tmp, 0, 0x5400);
+              break;
+    case 27:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[2] |= __byte_perm_S (tmp, 0, 0x4000);
+              break;
+    case 28:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              break;
+    case 29:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[3] |= tmp;
+              break;
+    case 30:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[3] |= __byte_perm_S (tmp, 0, 0x5400);
+              break;
+    case 31:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[2] |= __byte_perm_S (tmp, 0, 0x0000);
+              buf1[3] |= __byte_perm_S (tmp, 0, 0x4000);
+              break;
+  }
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  switch (p0)
+  {
+    case  1:  buf0[0] |= tmp <<  0;
+              break;
+    case  2:  buf0[0] |= tmp <<  0 | tmp << 8;
+              break;
+    case  3:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16;
+              break;
+    case  4:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              break;
+    case  5:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0;
+              break;
+    case  6:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8;
+              break;
+    case  7:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16;
+              break;
+    case  8:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              break;
+    case  9:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0;
+              break;
+    case 10:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8;
+              break;
+    case 11:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16;
+              break;
+    case 12:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              break;
+    case 13:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0;
+              break;
+    case 14:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8;
+              break;
+    case 15:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16;
+              break;
+    case 16:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              break;
+    case 17:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0;
+              break;
+    case 18:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8;
+              break;
+    case 19:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16;
+              break;
+    case 20:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              break;
+    case 21:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0;
+              break;
+    case 22:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8;
+              break;
+    case 23:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16;
+              break;
+    case 24:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              break;
+    case 25:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[2] |= tmp <<  0;
+              break;
+    case 26:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[2] |= tmp <<  0 | tmp << 8;
+              break;
+    case 27:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16;
+              break;
+    case 28:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              break;
+    case 29:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[3] |= tmp <<  0;
+              break;
+    case 30:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[3] |= tmp <<  0 | tmp << 8;
+              break;
+    case 31:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
+              buf1[3] |= tmp <<  0 | tmp << 8 | tmp << 16;
+              break;
+  }
+  #endif
+
+  out_len += p0;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_dupechar_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if ( in_len       ==  0) return (in_len);
+  if ((in_len + p0) >= 32) return (in_len);
+
+  const u32 in_len1 = in_len - 1;
+
+  const u32 sh = (in_len1 & 3) * 8;
+
+  u32 tmp = 0;
+
+  switch (in_len1 / 4)
+  {
+    case  0:  tmp = (buf0[0] >> sh) & 0xff; break;
+    case  1:  tmp = (buf0[1] >> sh) & 0xff; break;
+    case  2:  tmp = (buf0[2] >> sh) & 0xff; break;
+    case  3:  tmp = (buf0[3] >> sh) & 0xff; break;
+    case  4:  tmp = (buf1[0] >> sh) & 0xff; break;
+    case  5:  tmp = (buf1[1] >> sh) & 0xff; break;
+    case  6:  tmp = (buf1[2] >> sh) & 0xff; break;
+    case  7:  tmp = (buf1[3] >> sh) & 0xff; break;
+  }
+
+  u32 out_len = in_len;
+
+  for (u32 i = 0; i < p0; i++)
+  {
+    append_block1 (out_len, buf0, buf1, tmp);
+
+    out_len++;
+  }
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_dupechar_all (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if ( in_len           ==  0) return (in_len);
+  if ((in_len + in_len) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  #ifdef IS_NV
+  tib40[0] = __byte_perm_S (buf0[0], 0, 0x1100);
+  tib40[1] = __byte_perm_S (buf0[0], 0, 0x3322);
+  tib40[2] = __byte_perm_S (buf0[1], 0, 0x1100);
+  tib40[3] = __byte_perm_S (buf0[1], 0, 0x3322);
+  tib41[0] = __byte_perm_S (buf0[2], 0, 0x1100);
+  tib41[1] = __byte_perm_S (buf0[2], 0, 0x3322);
+  tib41[2] = __byte_perm_S (buf0[3], 0, 0x1100);
+  tib41[3] = __byte_perm_S (buf0[3], 0, 0x3322);
+
+  buf0[0] = tib40[0];
+  buf0[1] = tib40[1];
+  buf0[2] = tib40[2];
+  buf0[3] = tib40[3];
+  buf1[0] = tib41[0];
+  buf1[1] = tib41[1];
+  buf1[2] = tib41[2];
+  buf1[3] = tib41[3];
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  tib40[0] = ((buf0[0] & 0x000000FF) <<  0) | ((buf0[0] & 0x0000FF00) <<  8);
+  tib40[1] = ((buf0[0] & 0x00FF0000) >> 16) | ((buf0[0] & 0xFF000000) >>  8);
+  tib40[2] = ((buf0[1] & 0x000000FF) <<  0) | ((buf0[1] & 0x0000FF00) <<  8);
+  tib40[3] = ((buf0[1] & 0x00FF0000) >> 16) | ((buf0[1] & 0xFF000000) >>  8);
+  tib41[0] = ((buf0[2] & 0x000000FF) <<  0) | ((buf0[2] & 0x0000FF00) <<  8);
+  tib41[1] = ((buf0[2] & 0x00FF0000) >> 16) | ((buf0[2] & 0xFF000000) >>  8);
+  tib41[2] = ((buf0[3] & 0x000000FF) <<  0) | ((buf0[3] & 0x0000FF00) <<  8);
+  tib41[3] = ((buf0[3] & 0x00FF0000) >> 16) | ((buf0[3] & 0xFF000000) >>  8);
+
+  buf0[0] = tib40[0] | (tib40[0] <<  8);
+  buf0[1] = tib40[1] | (tib40[1] <<  8);
+  buf0[2] = tib40[2] | (tib40[2] <<  8);
+  buf0[3] = tib40[3] | (tib40[3] <<  8);
+  buf1[0] = tib41[0] | (tib41[0] <<  8);
+  buf1[1] = tib41[1] | (tib41[1] <<  8);
+  buf1[2] = tib41[2] | (tib41[2] <<  8);
+  buf1[3] = tib41[3] | (tib41[3] <<  8);
+  #endif
+
+  out_len = out_len + out_len;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_switch_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (in_len < 2) return (in_len);
+
+  #ifdef IS_NV
+  buf0[0] = __byte_perm_S (buf0[0], 0, 0x3201);
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  buf0[0] = (buf0[0] & 0xFFFF0000) | ((buf0[0] << 8) & 0x0000FF00) | ((buf0[0] >> 8) & 0x000000FF);
+  #endif
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_switch_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (in_len < 2) return (in_len);
+
+  #ifdef IS_NV
+  switch (in_len)
+  {
+    case  2:  buf0[0] = __byte_perm_S (buf0[0],       0, 0x5401);
+              break;
+    case  3:  buf0[0] = __byte_perm_S (buf0[0],       0, 0x4120);
+              break;
+    case  4:  buf0[0] = __byte_perm_S (buf0[0],       0, 0x2310);
+              break;
+    case  5:  buf0[1] = __byte_perm_S (buf0[1], buf0[0], 0x7210);
+              buf0[0] = __byte_perm_S (buf0[0], buf0[1], 0x4210);
+              buf0[1] = __byte_perm_S (buf0[1],       0, 0x6543);
+              break;
+    case  6:  buf0[1] = __byte_perm_S (buf0[1],       0, 0x5401);
+              break;
+    case  7:  buf0[1] = __byte_perm_S (buf0[1],       0, 0x4120);
+              break;
+    case  8:  buf0[1] = __byte_perm_S (buf0[1],       0, 0x2310);
+              break;
+    case  9:  buf0[2] = __byte_perm_S (buf0[2], buf0[1], 0x7210);
+              buf0[1] = __byte_perm_S (buf0[1], buf0[2], 0x4210);
+              buf0[2] = __byte_perm_S (buf0[2],       0, 0x6543);
+              break;
+    case 10:  buf0[2] = __byte_perm_S (buf0[2],       0, 0x5401);
+              break;
+    case 11:  buf0[2] = __byte_perm_S (buf0[2],       0, 0x4120);
+              break;
+    case 12:  buf0[2] = __byte_perm_S (buf0[2],       0, 0x2310);
+              break;
+    case 13:  buf0[3] = __byte_perm_S (buf0[3], buf0[2], 0x7210);
+              buf0[2] = __byte_perm_S (buf0[2], buf0[3], 0x4210);
+              buf0[3] = __byte_perm_S (buf0[3],       0, 0x6543);
+              break;
+    case 14:  buf0[3] = __byte_perm_S (buf0[3],       0, 0x5401);
+              break;
+    case 15:  buf0[3] = __byte_perm_S (buf0[3],       0, 0x4120);
+              break;
+    case 16:  buf0[3] = __byte_perm_S (buf0[3],       0, 0x2310);
+              break;
+    case 17:  buf1[0] = __byte_perm_S (buf1[0], buf0[3], 0x7210);
+              buf0[3] = __byte_perm_S (buf0[3], buf1[0], 0x4210);
+              buf1[0] = __byte_perm_S (buf1[0],       0, 0x6543);
+              break;
+    case 18:  buf1[0] = __byte_perm_S (buf1[0],       0, 0x5401);
+              break;
+    case 19:  buf1[0] = __byte_perm_S (buf1[0],       0, 0x4120);
+              break;
+    case 20:  buf1[0] = __byte_perm_S (buf1[0],       0, 0x2310);
+              break;
+    case 21:  buf1[1] = __byte_perm_S (buf1[1], buf1[0], 0x7210);
+              buf1[0] = __byte_perm_S (buf1[0], buf1[1], 0x4210);
+              buf1[1] = __byte_perm_S (buf1[1],       0, 0x6543);
+              break;
+    case 22:  buf1[1] = __byte_perm_S (buf1[1],       0, 0x5401);
+              break;
+    case 23:  buf1[1] = __byte_perm_S (buf1[1],       0, 0x4120);
+              break;
+    case 24:  buf1[1] = __byte_perm_S (buf1[1],       0, 0x2310);
+              break;
+    case 25:  buf1[2] = __byte_perm_S (buf1[2], buf1[1], 0x7210);
+              buf1[1] = __byte_perm_S (buf1[1], buf1[2], 0x4210);
+              buf1[2] = __byte_perm_S (buf1[2],       0, 0x6543);
+              break;
+    case 26:  buf1[2] = __byte_perm_S (buf1[2],       0, 0x5401);
+              break;
+    case 27:  buf1[2] = __byte_perm_S (buf1[2],       0, 0x4120);
+              break;
+    case 28:  buf1[2] = __byte_perm_S (buf1[2],       0, 0x2310);
+              break;
+    case 29:  buf1[3] = __byte_perm_S (buf1[3], buf1[2], 0x7210);
+              buf1[2] = __byte_perm_S (buf1[2], buf1[3], 0x4210);
+              buf1[3] = __byte_perm_S (buf1[3],       0, 0x6543);
+              break;
+    case 30:  buf1[3] = __byte_perm_S (buf1[3],       0, 0x5401);
+              break;
+    case 31:  buf1[3] = __byte_perm_S (buf1[3],       0, 0x4120);
+              break;
+  }
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  switch (in_len)
+  {
+    case  2:  buf0[0] = ((buf0[0] << 8) & 0x0000FF00) | ((buf0[0] >> 8) & 0x000000FF);
+              break;
+    case  3:  buf0[0] =  (buf0[0] & 0x000000FF) | ((buf0[0] <<  8) & 0x00FF0000) | ((buf0[0] >>  8) & 0x0000FF00);
+              break;
+    case  4:  buf0[0] =  (buf0[0] & 0x0000FFFF) | ((buf0[0] <<  8) & 0xFF000000) | ((buf0[0] >>  8) & 0x00FF0000);
+              break;
+    case  5:  buf0[1] =  (buf0[0] & 0xFF000000) |   buf0[1];
+              buf0[0] =  (buf0[0] & 0x00FFFFFF) |  (buf0[1] << 24);
+              buf0[1] =  (buf0[1] >> 24);
+              break;
+    case  6:  buf0[1] = ((buf0[1] << 8) & 0x0000FF00) | ((buf0[1] >> 8) & 0x000000FF);
+              break;
+    case  7:  buf0[1] =  (buf0[1] & 0x000000FF) | ((buf0[1] <<  8) & 0x00FF0000) | ((buf0[1] >>  8) & 0x0000FF00);
+              break;
+    case  8:  buf0[1] =  (buf0[1] & 0x0000FFFF) | ((buf0[1] <<  8) & 0xFF000000) | ((buf0[1] >>  8) & 0x00FF0000);
+              break;
+    case  9:  buf0[2] =  (buf0[1] & 0xFF000000) |   buf0[2];
+              buf0[1] =  (buf0[1] & 0x00FFFFFF) |  (buf0[2] << 24);
+              buf0[2] =  (buf0[2] >> 24);
+              break;
+    case 10:  buf0[2] = ((buf0[2] << 8) & 0x0000FF00) | ((buf0[2] >> 8) & 0x000000FF);
+              break;
+    case 11:  buf0[2] =  (buf0[2] & 0x000000FF) | ((buf0[2] <<  8) & 0x00FF0000) | ((buf0[2] >>  8) & 0x0000FF00);
+              break;
+    case 12:  buf0[2] =  (buf0[2] & 0x0000FFFF) | ((buf0[2] <<  8) & 0xFF000000) | ((buf0[2] >>  8) & 0x00FF0000);
+              break;
+    case 13:  buf0[3] =  (buf0[2] & 0xFF000000) |   buf0[3];
+              buf0[2] =  (buf0[2] & 0x00FFFFFF) |  (buf0[3] << 24);
+              buf0[3] =  (buf0[3] >> 24);
+              break;
+    case 14:  buf0[3] = ((buf0[3] << 8) & 0x0000FF00) | ((buf0[3] >> 8) & 0x000000FF);
+              break;
+    case 15:  buf0[3] =  (buf0[3] & 0x000000FF) |  ((buf0[3] <<  8) & 0x00FF0000) | ((buf0[3] >>  8) & 0x0000FF00);
+              break;
+    case 16:  buf0[3] =  (buf0[3] & 0x0000FFFF) | ((buf0[3] <<  8) & 0xFF000000) | ((buf0[3] >>  8) & 0x00FF0000);
+              break;
+    case 17:  buf1[0] =  (buf0[3] & 0xFF000000) |   buf1[0];
+              buf0[3] =  (buf0[3] & 0x00FFFFFF) |  (buf1[0] << 24);
+              buf1[0] =  (buf1[0] >> 24);
+              break;
+    case 18:  buf1[0] = ((buf1[0] << 8) & 0x0000FF00) | ((buf1[0] >> 8) & 0x000000FF);
+              break;
+    case 19:  buf1[0] =  (buf1[0] & 0x000000FF) | ((buf1[0] <<  8) & 0x00FF0000) | ((buf1[0] >>  8) & 0x0000FF00);
+              break;
+    case 20:  buf1[0] =  (buf1[0] & 0x0000FFFF) | ((buf1[0] <<  8) & 0xFF000000) | ((buf1[0] >>  8) & 0x00FF0000);
+              break;
+    case 21:  buf1[1] =  (buf1[0] & 0xFF000000) |   buf1[1];
+              buf1[0] =  (buf1[0] & 0x00FFFFFF) |  (buf1[1] << 24);
+              buf1[1] =  (buf1[1] >> 24);
+              break;
+    case 22:  buf1[1] = ((buf1[1] << 8) & 0x0000FF00) | ((buf1[1] >> 8) & 0x000000FF);
+              break;
+    case 23:  buf1[1] =  (buf1[1] & 0x000000FF) | ((buf1[1] <<  8) & 0x00FF0000) | ((buf1[1] >>  8) & 0x0000FF00);
+              break;
+    case 24:  buf1[1] =  (buf1[1] & 0x0000FFFF) | ((buf1[1] <<  8) & 0xFF000000) | ((buf1[1] >>  8) & 0x00FF0000);
+              break;
+    case 25:  buf1[2] =  (buf1[1] & 0xFF000000) |   buf1[2];
+              buf1[1] =  (buf1[1] & 0x00FFFFFF) |  (buf1[2] << 24);
+              buf1[2] =  (buf1[2] >> 24);
+              break;
+    case 26:  buf1[2] = ((buf1[2] << 8) & 0x0000FF00) | ((buf1[2] >> 8) & 0x000000FF);
+              break;
+    case 27:  buf1[2] =  (buf1[2] & 0x000000FF) | ((buf1[2] <<  8) & 0x00FF0000) | ((buf1[2] >>  8) & 0x0000FF00);
+              break;
+    case 28:  buf1[2] =  (buf1[2] & 0x0000FFFF) | ((buf1[2] <<  8) & 0xFF000000) | ((buf1[2] >>  8) & 0x00FF0000);
+              break;
+    case 29:  buf1[3] =  (buf1[2] & 0xFF000000) |   buf1[3];
+              buf1[2] =  (buf1[2] & 0x00FFFFFF) |  (buf1[3] << 24);
+              buf1[3] =  (buf1[3] >> 24);
+              break;
+    case 30:  buf1[3] = ((buf1[3] << 8) & 0x0000FF00) | ((buf1[3] >> 8) & 0x000000FF);
+              break;
+    case 31:  buf1[3] =  (buf1[3] & 0x000000FF) |  ((buf1[3] <<  8) & 0x00FF0000) | ((buf1[3] >>  8) & 0x0000FF00);
+              break;
+  }
+  #endif
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_switch_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+  if (p1 >= in_len) return (in_len);
+
+  u32 tmp0 = 0;
+  u32 tmp1 = 0;
+
+  #ifdef IS_NV
+  switch (p0)
+  {
+    case  0:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6540);
+              break;
+    case  1:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6541);
+              break;
+    case  2:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6542);
+              break;
+    case  3:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6543);
+              break;
+    case  4:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6540);
+              break;
+    case  5:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6541);
+              break;
+    case  6:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6542);
+              break;
+    case  7:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6543);
+              break;
+    case  8:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6540);
+              break;
+    case  9:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6541);
+              break;
+    case 10:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6542);
+              break;
+    case 11:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6543);
+              break;
+    case 12:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6540);
+              break;
+    case 13:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6541);
+              break;
+    case 14:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6542);
+              break;
+    case 15:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6543);
+              break;
+    case 16:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6540);
+              break;
+    case 17:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6541);
+              break;
+    case 18:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6542);
+              break;
+    case 19:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6543);
+              break;
+    case 20:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6540);
+              break;
+    case 21:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6541);
+              break;
+    case 22:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6542);
+              break;
+    case 23:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6543);
+              break;
+    case 24:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6540);
+              break;
+    case 25:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6541);
+              break;
+    case 26:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6542);
+              break;
+    case 27:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6543);
+              break;
+    case 28:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6540);
+              break;
+    case 29:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6541);
+              break;
+    case 30:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6542);
+              break;
+    case 31:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6543);
+              break;
+  }
+
+  switch (p1)
+  {
+    case  0:  tmp1    = __byte_perm_S (buf0[0],       0, 0x6540);
+              buf0[0] = __byte_perm_S (tmp0,    buf0[0], 0x7650);
+              break;
+    case  1:  tmp1    = __byte_perm_S (buf0[0],       0, 0x6541);
+              buf0[0] = __byte_perm_S (tmp0,    buf0[0], 0x7604);
+              break;
+    case  2:  tmp1    = __byte_perm_S (buf0[0],       0, 0x6542);
+              buf0[0] = __byte_perm_S (tmp0,    buf0[0], 0x7054);
+              break;
+    case  3:  tmp1    = __byte_perm_S (buf0[0],       0, 0x6543);
+              buf0[0] = __byte_perm_S (tmp0,    buf0[0], 0x0654);
+              break;
+    case  4:  tmp1    = __byte_perm_S (buf0[1],       0, 0x6540);
+              buf0[1] = __byte_perm_S (tmp0,    buf0[1], 0x7650);
+              break;
+    case  5:  tmp1    = __byte_perm_S (buf0[1],       0, 0x6541);
+              buf0[1] = __byte_perm_S (tmp0,    buf0[1], 0x7604);
+              break;
+    case  6:  tmp1    = __byte_perm_S (buf0[1],       0, 0x6542);
+              buf0[1] = __byte_perm_S (tmp0,    buf0[1], 0x7054);
+              break;
+    case  7:  tmp1    = __byte_perm_S (buf0[1],       0, 0x6543);
+              buf0[1] = __byte_perm_S (tmp0,    buf0[1], 0x0654);
+              break;
+    case  8:  tmp1    = __byte_perm_S (buf0[2],       0, 0x6540);
+              buf0[2] = __byte_perm_S (tmp0,    buf0[2], 0x7650);
+              break;
+    case  9:  tmp1    = __byte_perm_S (buf0[2],       0, 0x6541);
+              buf0[2] = __byte_perm_S (tmp0,    buf0[2], 0x7604);
+              break;
+    case 10:  tmp1    = __byte_perm_S (buf0[2],       0, 0x6542);
+              buf0[2] = __byte_perm_S (tmp0,    buf0[2], 0x7054);
+              break;
+    case 11:  tmp1    = __byte_perm_S (buf0[2],       0, 0x6543);
+              buf0[2] = __byte_perm_S (tmp0,    buf0[2], 0x0654);
+              break;
+    case 12:  tmp1    = __byte_perm_S (buf0[3],       0, 0x6540);
+              buf0[3] = __byte_perm_S (tmp0,    buf0[3], 0x7650);
+              break;
+    case 13:  tmp1    = __byte_perm_S (buf0[3],       0, 0x6541);
+              buf0[3] = __byte_perm_S (tmp0,    buf0[3], 0x7604);
+              break;
+    case 14:  tmp1    = __byte_perm_S (buf0[3],       0, 0x6542);
+              buf0[3] = __byte_perm_S (tmp0,    buf0[3], 0x7054);
+              break;
+    case 15:  tmp1    = __byte_perm_S (buf0[3],       0, 0x6543);
+              buf0[3] = __byte_perm_S (tmp0,    buf0[3], 0x0654);
+              break;
+    case 16:  tmp1    = __byte_perm_S (buf1[0],       0, 0x6540);
+              buf1[0] = __byte_perm_S (tmp0,    buf1[0], 0x7650);
+              break;
+    case 17:  tmp1    = __byte_perm_S (buf1[0],       0, 0x6541);
+              buf1[0] = __byte_perm_S (tmp0,    buf1[0], 0x7604);
+              break;
+    case 18:  tmp1    = __byte_perm_S (buf1[0],       0, 0x6542);
+              buf1[0] = __byte_perm_S (tmp0,    buf1[0], 0x7054);
+              break;
+    case 19:  tmp1    = __byte_perm_S (buf1[0],       0, 0x6543);
+              buf1[0] = __byte_perm_S (tmp0,    buf1[0], 0x0654);
+              break;
+    case 20:  tmp1    = __byte_perm_S (buf1[1],       0, 0x6540);
+              buf1[1] = __byte_perm_S (tmp0,    buf1[1], 0x7650);
+              break;
+    case 21:  tmp1    = __byte_perm_S (buf1[1],       0, 0x6541);
+              buf1[1] = __byte_perm_S (tmp0,    buf1[1], 0x7604);
+              break;
+    case 22:  tmp1    = __byte_perm_S (buf1[1],       0, 0x6542);
+              buf1[1] = __byte_perm_S (tmp0,    buf1[1], 0x7054);
+              break;
+    case 23:  tmp1    = __byte_perm_S (buf1[1],       0, 0x6543);
+              buf1[1] = __byte_perm_S (tmp0,    buf1[1], 0x0654);
+              break;
+    case 24:  tmp1    = __byte_perm_S (buf1[2],       0, 0x6540);
+              buf1[2] = __byte_perm_S (tmp0,    buf1[2], 0x7650);
+              break;
+    case 25:  tmp1    = __byte_perm_S (buf1[2],       0, 0x6541);
+              buf1[2] = __byte_perm_S (tmp0,    buf1[2], 0x7604);
+              break;
+    case 26:  tmp1    = __byte_perm_S (buf1[2],       0, 0x6542);
+              buf1[2] = __byte_perm_S (tmp0,    buf1[2], 0x7054);
+              break;
+    case 27:  tmp1    = __byte_perm_S (buf1[2],       0, 0x6543);
+              buf1[2] = __byte_perm_S (tmp0,    buf1[2], 0x0654);
+              break;
+    case 28:  tmp1    = __byte_perm_S (buf1[3],       0, 0x6540);
+              buf1[3] = __byte_perm_S (tmp0,    buf1[3], 0x7650);
+              break;
+    case 29:  tmp1    = __byte_perm_S (buf1[3],       0, 0x6541);
+              buf1[3] = __byte_perm_S (tmp0,    buf1[3], 0x7604);
+              break;
+    case 30:  tmp1    = __byte_perm_S (buf1[3],       0, 0x6542);
+              buf1[3] = __byte_perm_S (tmp0,    buf1[3], 0x7054);
+              break;
+    case 31:  tmp1    = __byte_perm_S (buf1[3],       0, 0x6543);
+              buf1[3] = __byte_perm_S (tmp0,    buf1[3], 0x0654);
+              break;
+  }
+
+  switch (p0)
+  {
+    case  0:  buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7650);
+              break;
+    case  1:  buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7604);
+              break;
+    case  2:  buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7054);
+              break;
+    case  3:  buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x0654);
+              break;
+    case  4:  buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7650);
+              break;
+    case  5:  buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7604);
+              break;
+    case  6:  buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7054);
+              break;
+    case  7:  buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x0654);
+              break;
+    case  8:  buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7650);
+              break;
+    case  9:  buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7604);
+              break;
+    case 10:  buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7054);
+              break;
+    case 11:  buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x0654);
+              break;
+    case 12:  buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7650);
+              break;
+    case 13:  buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7604);
+              break;
+    case 14:  buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7054);
+              break;
+    case 15:  buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x0654);
+              break;
+    case 16:  buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7650);
+              break;
+    case 17:  buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7604);
+              break;
+    case 18:  buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7054);
+              break;
+    case 19:  buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x0654);
+              break;
+    case 20:  buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7650);
+              break;
+    case 21:  buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7604);
+              break;
+    case 22:  buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7054);
+              break;
+    case 23:  buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x0654);
+              break;
+    case 24:  buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7650);
+              break;
+    case 25:  buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7604);
+              break;
+    case 26:  buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7054);
+              break;
+    case 27:  buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x0654);
+              break;
+    case 28:  buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7650);
+              break;
+    case 29:  buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7604);
+              break;
+    case 30:  buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7054);
+              break;
+    case 31:  buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x0654);
+              break;
+  }
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  switch (p0)
+  {
+    case  0:  tmp0 = (buf0[0] >>  0) & 0xFF;
+              break;
+    case  1:  tmp0 = (buf0[0] >>  8) & 0xFF;
+              break;
+    case  2:  tmp0 = (buf0[0] >> 16) & 0xFF;
+              break;
+    case  3:  tmp0 = (buf0[0] >> 24) & 0xFF;
+              break;
+    case  4:  tmp0 = (buf0[1] >>  0) & 0xFF;
+              break;
+    case  5:  tmp0 = (buf0[1] >>  8) & 0xFF;
+              break;
+    case  6:  tmp0 = (buf0[1] >> 16) & 0xFF;
+              break;
+    case  7:  tmp0 = (buf0[1] >> 24) & 0xFF;
+              break;
+    case  8:  tmp0 = (buf0[2] >>  0) & 0xFF;
+              break;
+    case  9:  tmp0 = (buf0[2] >>  8) & 0xFF;
+              break;
+    case 10:  tmp0 = (buf0[2] >> 16) & 0xFF;
+              break;
+    case 11:  tmp0 = (buf0[2] >> 24) & 0xFF;
+              break;
+    case 12:  tmp0 = (buf0[3] >>  0) & 0xFF;
+              break;
+    case 13:  tmp0 = (buf0[3] >>  8) & 0xFF;
+              break;
+    case 14:  tmp0 = (buf0[3] >> 16) & 0xFF;
+              break;
+    case 15:  tmp0 = (buf0[3] >> 24) & 0xFF;
+              break;
+    case 16:  tmp0 = (buf1[0] >>  0) & 0xFF;
+              break;
+    case 17:  tmp0 = (buf1[0] >>  8) & 0xFF;
+              break;
+    case 18:  tmp0 = (buf1[0] >> 16) & 0xFF;
+              break;
+    case 19:  tmp0 = (buf1[0] >> 24) & 0xFF;
+              break;
+    case 20:  tmp0 = (buf1[1] >>  0) & 0xFF;
+              break;
+    case 21:  tmp0 = (buf1[1] >>  8) & 0xFF;
+              break;
+    case 22:  tmp0 = (buf1[1] >> 16) & 0xFF;
+              break;
+    case 23:  tmp0 = (buf1[1] >> 24) & 0xFF;
+              break;
+    case 24:  tmp0 = (buf1[2] >>  0) & 0xFF;
+              break;
+    case 25:  tmp0 = (buf1[2] >>  8) & 0xFF;
+              break;
+    case 26:  tmp0 = (buf1[2] >> 16) & 0xFF;
+              break;
+    case 27:  tmp0 = (buf1[2] >> 24) & 0xFF;
+              break;
+    case 28:  tmp0 = (buf1[3] >>  0) & 0xFF;
+              break;
+    case 29:  tmp0 = (buf1[3] >>  8) & 0xFF;
+              break;
+    case 30:  tmp0 = (buf1[3] >> 16) & 0xFF;
+              break;
+    case 31:  tmp0 = (buf1[3] >> 24) & 0xFF;
+              break;
+  }
+
+  switch (p1)
+  {
+    case  0:  tmp1 = (buf0[0] >>  0) & 0xff;
+              buf0[0]  = (buf0[0] & 0xffffff00) | tmp0 <<  0;
+              break;
+    case  1:  tmp1 = (buf0[0] >>  8) & 0xff;
+              buf0[0]  = (buf0[0] & 0xffff00ff) | tmp0 <<  8;
+              break;
+    case  2:  tmp1 = (buf0[0] >> 16) & 0xff;
+              buf0[0]  = (buf0[0] & 0xff00ffff) | tmp0 << 16;
+              break;
+    case  3:  tmp1 = (buf0[0] >> 24) & 0xff;
+              buf0[0]  = (buf0[0] & 0x00ffffff) | tmp0 << 24;
+              break;
+    case  4:  tmp1 = (buf0[1] >>  0) & 0xff;
+              buf0[1]  = (buf0[1] & 0xffffff00) | tmp0 <<  0;
+              break;
+    case  5:  tmp1 = (buf0[1] >>  8) & 0xff;
+              buf0[1]  = (buf0[1] & 0xffff00ff) | tmp0 <<  8;
+              break;
+    case  6:  tmp1 = (buf0[1] >> 16) & 0xff;
+              buf0[1]  = (buf0[1] & 0xff00ffff) | tmp0 << 16;
+              break;
+    case  7:  tmp1 = (buf0[1] >> 24) & 0xff;
+              buf0[1]  = (buf0[1] & 0x00ffffff) | tmp0 << 24;
+              break;
+    case  8:  tmp1 = (buf0[2] >>  0) & 0xff;
+              buf0[2]  = (buf0[2] & 0xffffff00) | tmp0 <<  0;
+              break;
+    case  9:  tmp1 = (buf0[2] >>  8) & 0xff;
+              buf0[2]  = (buf0[2] & 0xffff00ff) | tmp0 <<  8;
+              break;
+    case 10:  tmp1 = (buf0[2] >> 16) & 0xff;
+              buf0[2]  = (buf0[2] & 0xff00ffff) | tmp0 << 16;
+              break;
+    case 11:  tmp1 = (buf0[2] >> 24) & 0xff;
+              buf0[2]  = (buf0[2] & 0x00ffffff) | tmp0 << 24;
+              break;
+    case 12:  tmp1 = (buf0[3] >>  0) & 0xff;
+              buf0[3]  = (buf0[3] & 0xffffff00) | tmp0 <<  0;
+              break;
+    case 13:  tmp1 = (buf0[3] >>  8) & 0xff;
+              buf0[3]  = (buf0[3] & 0xffff00ff) | tmp0 <<  8;
+              break;
+    case 14:  tmp1 = (buf0[3] >> 16) & 0xff;
+              buf0[3]  = (buf0[3] & 0xff00ffff) | tmp0 << 16;
+              break;
+    case 15:  tmp1 = (buf0[3] >> 24) & 0xff;
+              buf0[3]  = (buf0[3] & 0x00ffffff) | tmp0 << 24;
+              break;
+    case 16:  tmp1 = (buf1[0] >>  0) & 0xff;
+              buf1[0]  = (buf1[0] & 0xffffff00) | tmp0 <<  0;
+              break;
+    case 17:  tmp1 = (buf1[0] >>  8) & 0xff;
+              buf1[0]  = (buf1[0] & 0xffff00ff) | tmp0 <<  8;
+              break;
+    case 18:  tmp1 = (buf1[0] >> 16) & 0xff;
+              buf1[0]  = (buf1[0] & 0xff00ffff) | tmp0 << 16;
+              break;
+    case 19:  tmp1 = (buf1[0] >> 24) & 0xff;
+              buf1[0]  = (buf1[0] & 0x00ffffff) | tmp0 << 24;
+              break;
+    case 20:  tmp1 = (buf1[1] >>  0) & 0xff;
+              buf1[1]  = (buf1[1] & 0xffffff00) | tmp0 <<  0;
+              break;
+    case 21:  tmp1 = (buf1[1] >>  8) & 0xff;
+              buf1[1]  = (buf1[1] & 0xffff00ff) | tmp0 <<  8;
+              break;
+    case 22:  tmp1 = (buf1[1] >> 16) & 0xff;
+              buf1[1]  = (buf1[1] & 0xff00ffff) | tmp0 << 16;
+              break;
+    case 23:  tmp1 = (buf1[1] >> 24) & 0xff;
+              buf1[1]  = (buf1[1] & 0x00ffffff) | tmp0 << 24;
+              break;
+    case 24:  tmp1 = (buf1[2] >>  0) & 0xff;
+              buf1[2]  = (buf1[2] & 0xffffff00) | tmp0 <<  0;
+              break;
+    case 25:  tmp1 = (buf1[2] >>  8) & 0xff;
+              buf1[2]  = (buf1[2] & 0xffff00ff) | tmp0 <<  8;
+              break;
+    case 26:  tmp1 = (buf1[2] >> 16) & 0xff;
+              buf1[2]  = (buf1[2] & 0xff00ffff) | tmp0 << 16;
+              break;
+    case 27:  tmp1 = (buf1[2] >> 24) & 0xff;
+              buf1[2]  = (buf1[2] & 0x00ffffff) | tmp0 << 24;
+              break;
+    case 28:  tmp1 = (buf1[3] >>  0) & 0xff;
+              buf1[3]  = (buf1[3] & 0xffffff00) | tmp0 <<  0;
+              break;
+    case 29:  tmp1 = (buf1[3] >>  8) & 0xff;
+              buf1[3]  = (buf1[3] & 0xffff00ff) | tmp0 <<  8;
+              break;
+    case 30:  tmp1 = (buf1[3] >> 16) & 0xff;
+              buf1[3]  = (buf1[3] & 0xff00ffff) | tmp0 << 16;
+              break;
+    case 31:  tmp1 = (buf1[3] >> 24) & 0xff;
+              buf1[3]  = (buf1[3] & 0x00ffffff) | tmp0 << 24;
+              break;
+  }
+
+  switch (p0)
+  {
+    case  0:  buf0[0]  = (buf0[0] & 0xffffff00) | tmp1 <<  0;
+              break;
+    case  1:  buf0[0]  = (buf0[0] & 0xffff00ff) | tmp1 <<  8;
+              break;
+    case  2:  buf0[0]  = (buf0[0] & 0xff00ffff) | tmp1 << 16;
+              break;
+    case  3:  buf0[0]  = (buf0[0] & 0x00ffffff) | tmp1 << 24;
+              break;
+    case  4:  buf0[1]  = (buf0[1] & 0xffffff00) | tmp1 <<  0;
+              break;
+    case  5:  buf0[1]  = (buf0[1] & 0xffff00ff) | tmp1 <<  8;
+              break;
+    case  6:  buf0[1]  = (buf0[1] & 0xff00ffff) | tmp1 << 16;
+              break;
+    case  7:  buf0[1]  = (buf0[1] & 0x00ffffff) | tmp1 << 24;
+              break;
+    case  8:  buf0[2]  = (buf0[2] & 0xffffff00) | tmp1 <<  0;
+              break;
+    case  9:  buf0[2]  = (buf0[2] & 0xffff00ff) | tmp1 <<  8;
+              break;
+    case 10:  buf0[2]  = (buf0[2] & 0xff00ffff) | tmp1 << 16;
+              break;
+    case 11:  buf0[2]  = (buf0[2] & 0x00ffffff) | tmp1 << 24;
+              break;
+    case 12:  buf0[3]  = (buf0[3] & 0xffffff00) | tmp1 <<  0;
+              break;
+    case 13:  buf0[3]  = (buf0[3] & 0xffff00ff) | tmp1 <<  8;
+              break;
+    case 14:  buf0[3]  = (buf0[3] & 0xff00ffff) | tmp1 << 16;
+              break;
+    case 15:  buf0[3]  = (buf0[3] & 0x00ffffff) | tmp1 << 24;
+              break;
+    case 16:  buf1[0]  = (buf1[0] & 0xffffff00) | tmp1 <<  0;
+              break;
+    case 17:  buf1[0]  = (buf1[0] & 0xffff00ff) | tmp1 <<  8;
+              break;
+    case 18:  buf1[0]  = (buf1[0] & 0xff00ffff) | tmp1 << 16;
+              break;
+    case 19:  buf1[0]  = (buf1[0] & 0x00ffffff) | tmp1 << 24;
+              break;
+    case 20:  buf1[1]  = (buf1[1] & 0xffffff00) | tmp1 <<  0;
+              break;
+    case 21:  buf1[1]  = (buf1[1] & 0xffff00ff) | tmp1 <<  8;
+              break;
+    case 22:  buf1[1]  = (buf1[1] & 0xff00ffff) | tmp1 << 16;
+              break;
+    case 23:  buf1[1]  = (buf1[1] & 0x00ffffff) | tmp1 << 24;
+              break;
+    case 24:  buf1[2]  = (buf1[2] & 0xffffff00) | tmp1 <<  0;
+              break;
+    case 25:  buf1[2]  = (buf1[2] & 0xffff00ff) | tmp1 <<  8;
+              break;
+    case 26:  buf1[2]  = (buf1[2] & 0xff00ffff) | tmp1 << 16;
+              break;
+    case 27:  buf1[2]  = (buf1[2] & 0x00ffffff) | tmp1 << 24;
+              break;
+    case 28:  buf1[3]  = (buf1[3] & 0xffffff00) | tmp1 <<  0;
+              break;
+    case 29:  buf1[3]  = (buf1[3] & 0xffff00ff) | tmp1 <<  8;
+              break;
+    case 30:  buf1[3]  = (buf1[3] & 0xff00ffff) | tmp1 << 16;
+              break;
+    case 31:  buf1[3]  = (buf1[3] & 0x00ffffff) | tmp1 << 24;
+              break;
+  }
+  #endif
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_chr_shiftl (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  const u32 mr = 0xffu << ((p0 & 3) * 8);
+  const u32 ml = ~mr;
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) << 1) & mr); break;
+    case  1:  buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) << 1) & mr); break;
+    case  2:  buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) << 1) & mr); break;
+    case  3:  buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) << 1) & mr); break;
+    case  4:  buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) << 1) & mr); break;
+    case  5:  buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) << 1) & mr); break;
+    case  6:  buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) << 1) & mr); break;
+    case  7:  buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) << 1) & mr); break;
+  }
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_chr_shiftr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  const u32 mr = 0xffu << ((p0 & 3) * 8);
+  const u32 ml = ~mr;
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) >> 1) & mr); break;
+    case  1:  buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) >> 1) & mr); break;
+    case  2:  buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) >> 1) & mr); break;
+    case  3:  buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) >> 1) & mr); break;
+    case  4:  buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) >> 1) & mr); break;
+    case  5:  buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) >> 1) & mr); break;
+    case  6:  buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) >> 1) & mr); break;
+    case  7:  buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) >> 1) & mr); break;
+  }
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_chr_incr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  const u32 mr = 0xffu << ((p0 & 3) * 8);
+  const u32 ml = ~mr;
+
+  const u32 n = 0x01010101 & mr;
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) + n) & mr); break;
+    case  1:  buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) + n) & mr); break;
+    case  2:  buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) + n) & mr); break;
+    case  3:  buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) + n) & mr); break;
+    case  4:  buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) + n) & mr); break;
+    case  5:  buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) + n) & mr); break;
+    case  6:  buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) + n) & mr); break;
+    case  7:  buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) + n) & mr); break;
+  }
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_chr_decr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 >= in_len) return (in_len);
+
+  const u32 mr = 0xffu << ((p0 & 3) * 8);
+  const u32 ml = ~mr;
+
+  const u32 n = 0x01010101 & mr;
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) - n) & mr); break;
+    case  1:  buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) - n) & mr); break;
+    case  2:  buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) - n) & mr); break;
+    case  3:  buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) - n) & mr); break;
+    case  4:  buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) - n) & mr); break;
+    case  5:  buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) - n) & mr); break;
+    case  6:  buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) - n) & mr); break;
+    case  7:  buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) - n) & mr); break;
+  }
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_replace_np1 (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if ((p0 + 1) >= in_len) return (in_len);
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  lshift_block (buf0, buf1, tib40, tib41);
+
+  const u32 mr = 0xffu << ((p0 & 3) * 8);
+  const u32 ml = ~mr;
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] = (buf0[0] & ml) | (tib40[0] & mr); break;
+    case  1:  buf0[1] = (buf0[1] & ml) | (tib40[1] & mr); break;
+    case  2:  buf0[2] = (buf0[2] & ml) | (tib40[2] & mr); break;
+    case  3:  buf0[3] = (buf0[3] & ml) | (tib40[3] & mr); break;
+    case  4:  buf1[0] = (buf1[0] & ml) | (tib41[0] & mr); break;
+    case  5:  buf1[1] = (buf1[1] & ml) | (tib41[1] & mr); break;
+    case  6:  buf1[2] = (buf1[2] & ml) | (tib41[2] & mr); break;
+    case  7:  buf1[3] = (buf1[3] & ml) | (tib41[3] & mr); break;
+  }
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_replace_nm1 (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 == 0) return (in_len);
+
+  if (p0 >= in_len) return (in_len);
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  rshift_block (buf0, buf1, tib40, tib41);
+
+  const u32 mr = 0xffu << ((p0 & 3) * 8);
+  const u32 ml = ~mr;
+
+  switch (p0 / 4)
+  {
+    case  0:  buf0[0] = (buf0[0] & ml) | (tib40[0] & mr); break;
+    case  1:  buf0[1] = (buf0[1] & ml) | (tib40[1] & mr); break;
+    case  2:  buf0[2] = (buf0[2] & ml) | (tib40[2] & mr); break;
+    case  3:  buf0[3] = (buf0[3] & ml) | (tib40[3] & mr); break;
+    case  4:  buf1[0] = (buf1[0] & ml) | (tib41[0] & mr); break;
+    case  5:  buf1[1] = (buf1[1] & ml) | (tib41[1] & mr); break;
+    case  6:  buf1[2] = (buf1[2] & ml) | (tib41[2] & mr); break;
+    case  7:  buf1[3] = (buf1[3] & ml) | (tib41[3] & mr); break;
+  }
+
+  return in_len;
+}
+
+inline u32 rule_op_mangle_dupeblock_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 > in_len) return (in_len);
+
+  if ((in_len + p0) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  tib40[0] = buf0[0];
+  tib40[1] = buf0[1];
+  tib40[2] = buf0[2];
+  tib40[3] = buf0[3];
+  tib41[0] = buf1[0];
+  tib41[1] = buf1[1];
+  tib41[2] = buf1[2];
+  tib41[3] = buf1[3];
+
+  truncate_right (tib40, tib41, p0);
+
+  rshift_block_N (buf0, buf1, buf0, buf1, p0);
+
+  buf0[0] |= tib40[0];
+  buf0[1] |= tib40[1];
+  buf0[2] |= tib40[2];
+  buf0[3] |= tib40[3];
+  buf1[0] |= tib41[0];
+  buf1[1] |= tib41[1];
+  buf1[2] |= tib41[2];
+  buf1[3] |= tib41[3];
+
+  out_len += p0;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_dupeblock_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  if (p0 > in_len) return (in_len);
+
+  if ((in_len + p0) >= 32) return (in_len);
+
+  u32 out_len = in_len;
+
+  u32 tib40[4];
+  u32 tib41[4];
+
+  rshift_block_N (buf0, buf1, tib40, tib41, p0);
+
+  truncate_left (tib40, tib41, out_len);
+
+  buf0[0] |= tib40[0];
+  buf0[1] |= tib40[1];
+  buf0[2] |= tib40[2];
+  buf0[3] |= tib40[3];
+  buf1[0] |= tib41[0];
+  buf1[1] |= tib41[1];
+  buf1[2] |= tib41[2];
+  buf1[3] |= tib41[3];
+
+  out_len += p0;
+
+  return out_len;
+}
+
+inline u32 rule_op_mangle_title (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  buf0[0] |= (generate_cmask (buf0[0]));
+  buf0[1] |= (generate_cmask (buf0[1]));
+  buf0[2] |= (generate_cmask (buf0[2]));
+  buf0[3] |= (generate_cmask (buf0[3]));
+  buf1[0] |= (generate_cmask (buf1[0]));
+  buf1[1] |= (generate_cmask (buf1[1]));
+  buf1[2] |= (generate_cmask (buf1[2]));
+  buf1[3] |= (generate_cmask (buf1[3]));
+
+  #ifdef IS_NV
+  buf0[0] &= ~(0x00000020 & generate_cmask (buf0[0]));
+
+  for (u32 i = 0; i < in_len; i++)
+  {
+    u32 tmp0;
+    u32 tmp1;
+
+    switch (i)
+    {
+      case  0:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6540);
+                tmp1 = ~(0x00002000 & generate_cmask (buf0[0])); break;
+      case  1:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6541);
+                tmp1 = ~(0x00200000 & generate_cmask (buf0[0])); break;
+      case  2:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6542);
+                tmp1 = ~(0x20000000 & generate_cmask (buf0[0])); break;
+      case  3:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6543);
+                tmp1 = ~(0x00000020 & generate_cmask (buf0[1])); break;
+      case  4:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6540);
+                tmp1 = ~(0x00002000 & generate_cmask (buf0[1])); break;
+      case  5:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6541);
+                tmp1 = ~(0x00200000 & generate_cmask (buf0[1])); break;
+      case  6:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6542);
+                tmp1 = ~(0x20000000 & generate_cmask (buf0[1])); break;
+      case  7:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6543);
+                tmp1 = ~(0x00000020 & generate_cmask (buf0[2])); break;
+      case  8:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6540);
+                tmp1 = ~(0x00002000 & generate_cmask (buf0[2])); break;
+      case  9:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6541);
+                tmp1 = ~(0x00200000 & generate_cmask (buf0[2])); break;
+      case 10:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6542);
+                tmp1 = ~(0x20000000 & generate_cmask (buf0[2])); break;
+      case 11:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6543);
+                tmp1 = ~(0x00000020 & generate_cmask (buf0[3])); break;
+      case 12:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6540);
+                tmp1 = ~(0x00002000 & generate_cmask (buf0[3])); break;
+      case 13:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6541);
+                tmp1 = ~(0x00200000 & generate_cmask (buf0[3])); break;
+      case 14:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6542);
+                tmp1 = ~(0x20000000 & generate_cmask (buf0[3])); break;
+      case 15:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6543);
+                tmp1 = ~(0x00000020 & generate_cmask (buf1[0])); break;
+      case 16:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6540);
+                tmp1 = ~(0x00002000 & generate_cmask (buf1[0])); break;
+      case 17:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6541);
+                tmp1 = ~(0x00200000 & generate_cmask (buf1[0])); break;
+      case 18:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6542);
+                tmp1 = ~(0x20000000 & generate_cmask (buf1[0])); break;
+      case 19:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6543);
+                tmp1 = ~(0x00000020 & generate_cmask (buf1[1])); break;
+      case 20:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6540);
+                tmp1 = ~(0x00002000 & generate_cmask (buf1[1])); break;
+      case 21:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6541);
+                tmp1 = ~(0x00200000 & generate_cmask (buf1[1])); break;
+      case 22:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6542);
+                tmp1 = ~(0x20000000 & generate_cmask (buf1[1])); break;
+      case 23:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6543);
+                tmp1 = ~(0x00000020 & generate_cmask (buf1[2])); break;
+      case 24:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6540);
+                tmp1 = ~(0x00002000 & generate_cmask (buf1[2])); break;
+      case 25:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6541);
+                tmp1 = ~(0x00200000 & generate_cmask (buf1[2])); break;
+      case 26:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6542);
+                tmp1 = ~(0x20000000 & generate_cmask (buf1[2])); break;
+      case 27:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6543);
+                tmp1 = ~(0x00000020 & generate_cmask (buf1[3])); break;
+      case 28:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6540);
+                tmp1 = ~(0x00002000 & generate_cmask (buf1[3])); break;
+      case 29:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6541);
+                tmp1 = ~(0x00200000 & generate_cmask (buf1[3])); break;
+      case 30:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6542);
+                tmp1 = ~(0x20000000 & generate_cmask (buf1[3])); break;
+    }
+
+    if (i < 3)
+    {
+      if (tmp0   == ' ') buf0[0]   &= tmp1  ;
+    }
+    else if (i < 7)
+    {
+      if (tmp0   == ' ') buf0[1]   &= tmp1  ;
+    }
+    else if (i < 11)
+    {
+      if (tmp0   == ' ') buf0[2]   &= tmp1  ;
+    }
+    else if (i < 15)
+    {
+      if (tmp0   == ' ') buf0[3]   &= tmp1  ;
+    }
+    else if (i < 19)
+    {
+      if (tmp0   == ' ') buf1[0]   &= tmp1  ;
+    }
+    else if (i < 23)
+    {
+      if (tmp0   == ' ') buf1[1]   &= tmp1  ;
+    }
+    else if (i < 27)
+    {
+      if (tmp0   == ' ') buf1[2]   &= tmp1  ;
+    }
+    else if (i < 31)
+    {
+      if (tmp0   == ' ') buf1[3]   &= tmp1  ;
+    }
+  }
+  #endif
+
+  #if defined IS_AMD || defined IS_GENERIC
+  u32 tib40[4];
+  u32 tib41[4];
+
+  const uchar4 tmp0 = (uchar4) (' ');
+  const uchar4 tmp1 = (uchar4) (0x00);
+  const uchar4 tmp2 = (uchar4) (0xff);
+
+  uchar4 tmp;
+
+  tmp = as_uchar4 (buf0[0]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[0] = as_uint (tmp);
+  tmp = as_uchar4 (buf0[1]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[1] = as_uint (tmp);
+  tmp = as_uchar4 (buf0[2]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[2] = as_uint (tmp);
+  tmp = as_uchar4 (buf0[3]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[3] = as_uint (tmp);
+  tmp = as_uchar4 (buf1[0]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[0] = as_uint (tmp);
+  tmp = as_uchar4 (buf1[1]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[1] = as_uint (tmp);
+  tmp = as_uchar4 (buf1[2]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[2] = as_uint (tmp);
+  tmp = as_uchar4 (buf1[3]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[3] = as_uint (tmp);
+
+  rshift_block (tib40, tib41, tib40, tib41); tib40[0] |= 0xff;
+
+  buf0[0] &= ~(generate_cmask (buf0[0]) & tib40[0]);
+  buf0[1] &= ~(generate_cmask (buf0[1]) & tib40[1]);
+  buf0[2] &= ~(generate_cmask (buf0[2]) & tib40[2]);
+  buf0[3] &= ~(generate_cmask (buf0[3]) & tib40[3]);
+  buf1[0] &= ~(generate_cmask (buf1[0]) & tib41[0]);
+  buf1[1] &= ~(generate_cmask (buf1[1]) & tib41[1]);
+  buf1[2] &= ~(generate_cmask (buf1[2]) & tib41[2]);
+  buf1[3] &= ~(generate_cmask (buf1[3]) & tib41[3]);
+  #endif
+
+  return in_len;
+}
+
+inline u32 apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
+{
+  u32 out_len = in_len;
+
+  switch (name)
+  {
+    case RULE_OP_MANGLE_LREST:            out_len = rule_op_mangle_lrest            (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_UREST:            out_len = rule_op_mangle_urest            (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_LREST_UFIRST:     out_len = rule_op_mangle_lrest_ufirst     (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_UREST_LFIRST:     out_len = rule_op_mangle_urest_lfirst     (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_TREST:            out_len = rule_op_mangle_trest            (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_TOGGLE_AT:        out_len = rule_op_mangle_toggle_at        (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_REVERSE:          out_len = rule_op_mangle_reverse          (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DUPEWORD:         out_len = rule_op_mangle_dupeword         (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DUPEWORD_TIMES:   out_len = rule_op_mangle_dupeword_times   (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_REFLECT:          out_len = rule_op_mangle_reflect          (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_APPEND:           out_len = rule_op_mangle_append           (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_PREPEND:          out_len = rule_op_mangle_prepend          (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_ROTATE_LEFT:      out_len = rule_op_mangle_rotate_left      (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_ROTATE_RIGHT:     out_len = rule_op_mangle_rotate_right     (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DELETE_FIRST:     out_len = rule_op_mangle_delete_first     (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DELETE_LAST:      out_len = rule_op_mangle_delete_last      (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DELETE_AT:        out_len = rule_op_mangle_delete_at        (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_EXTRACT:          out_len = rule_op_mangle_extract          (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_OMIT:             out_len = rule_op_mangle_omit             (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_INSERT:           out_len = rule_op_mangle_insert           (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_OVERSTRIKE:       out_len = rule_op_mangle_overstrike       (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_TRUNCATE_AT:      out_len = rule_op_mangle_truncate_at      (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_REPLACE:          out_len = rule_op_mangle_replace          (p0, p1, buf0, buf1, out_len); break;
+    //case RULE_OP_MANGLE_PURGECHAR:        out_len = rule_op_mangle_purgechar        (p0, p1, buf0, buf1, out_len); break;
+    //case RULE_OP_MANGLE_TOGGLECASE_REC:   out_len = rule_op_mangle_togglecase_rec   (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DUPECHAR_FIRST:   out_len = rule_op_mangle_dupechar_first   (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DUPECHAR_LAST:    out_len = rule_op_mangle_dupechar_last    (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DUPECHAR_ALL:     out_len = rule_op_mangle_dupechar_all     (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_SWITCH_FIRST:     out_len = rule_op_mangle_switch_first     (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_SWITCH_LAST:      out_len = rule_op_mangle_switch_last      (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_SWITCH_AT:        out_len = rule_op_mangle_switch_at        (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_CHR_SHIFTL:       out_len = rule_op_mangle_chr_shiftl       (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_CHR_SHIFTR:       out_len = rule_op_mangle_chr_shiftr       (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_CHR_INCR:         out_len = rule_op_mangle_chr_incr         (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_CHR_DECR:         out_len = rule_op_mangle_chr_decr         (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_REPLACE_NP1:      out_len = rule_op_mangle_replace_np1      (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_REPLACE_NM1:      out_len = rule_op_mangle_replace_nm1      (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DUPEBLOCK_FIRST:  out_len = rule_op_mangle_dupeblock_first  (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_DUPEBLOCK_LAST:   out_len = rule_op_mangle_dupeblock_last   (p0, p1, buf0, buf1, out_len); break;
+    case RULE_OP_MANGLE_TITLE:            out_len = rule_op_mangle_title            (p0, p1, buf0, buf1, out_len); break;
+  }
+
+  return out_len;
+}
+
+inline u32 apply_rules (const __global u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len)
+{
+  u32 out_len = len;
+
+  for (u32 i = 0; cmds[i] != 0; i++)
+  {
+    const u32 cmd = cmds[i];
+
+    const u32 name = (cmd >>  0) & 0xff;
+    const u32 p0   = (cmd >>  8) & 0xff;
+    const u32 p1   = (cmd >> 16) & 0xff;
+
+    out_len = apply_rule (name, p0, p1, buf0, buf1, out_len);
+  }
+
+  return out_len;
+}
+
+inline u32x apply_rules_vect (const u32 pw_buf0[4], const u32 pw_buf1[4], const u32 pw_len, const __global kernel_rule_t *rules_buf, const u32 il_pos, u32x w0[4], u32x w1[4])
+{
+  #if VECT_SIZE == 1
+
+  w0[0] = pw_buf0[0];
+  w0[1] = pw_buf0[1];
+  w0[2] = pw_buf0[2];
+  w0[3] = pw_buf0[3];
+  w1[0] = pw_buf1[0];
+  w1[1] = pw_buf1[1];
+  w1[2] = pw_buf1[2];
+  w1[3] = pw_buf1[3];
+
+  return apply_rules (rules_buf[il_pos].cmds, w0, w1, pw_len);
+
+  #else
+
+  u32x out_len = 0;
+
+  #ifdef _unroll
+  #pragma unroll
+  #endif
+  for (int i = 0; i < VECT_SIZE; i++)
+  {
+    u32 tmp0[4];
+    u32 tmp1[4];
+
+    tmp0[0] = pw_buf0[0];
+    tmp0[1] = pw_buf0[1];
+    tmp0[2] = pw_buf0[2];
+    tmp0[3] = pw_buf0[3];
+    tmp1[0] = pw_buf1[0];
+    tmp1[1] = pw_buf1[1];
+    tmp1[2] = pw_buf1[2];
+    tmp1[3] = pw_buf1[3];
+
+    const u32 tmp_len = apply_rules (rules_buf[il_pos + i].cmds, tmp0, tmp1, pw_len);
+
+    switch (i)
+    {
+      #if VECT_SIZE >= 2
+      case 0:
+        w0[0].s0 = tmp0[0];
+        w0[1].s0 = tmp0[1];
+        w0[2].s0 = tmp0[2];
+        w0[3].s0 = tmp0[3];
+        w1[0].s0 = tmp1[0];
+        w1[1].s0 = tmp1[1];
+        w1[2].s0 = tmp1[2];
+        w1[3].s0 = tmp1[3];
+        out_len.s0 = tmp_len;
+        break;
+
+      case 1:
+        w0[0].s1 = tmp0[0];
+        w0[1].s1 = tmp0[1];
+        w0[2].s1 = tmp0[2];
+        w0[3].s1 = tmp0[3];
+        w1[0].s1 = tmp1[0];
+        w1[1].s1 = tmp1[1];
+        w1[2].s1 = tmp1[2];
+        w1[3].s1 = tmp1[3];
+        out_len.s1 = tmp_len;
+        break;
+      #endif
+
+      #if VECT_SIZE >= 4
+      case 2:
+        w0[0].s2 = tmp0[0];
+        w0[1].s2 = tmp0[1];
+        w0[2].s2 = tmp0[2];
+        w0[3].s2 = tmp0[3];
+        w1[0].s2 = tmp1[0];
+        w1[1].s2 = tmp1[1];
+        w1[2].s2 = tmp1[2];
+        w1[3].s2 = tmp1[3];
+        out_len.s2 = tmp_len;
+        break;
+
+      case 3:
+        w0[0].s3 = tmp0[0];
+        w0[1].s3 = tmp0[1];
+        w0[2].s3 = tmp0[2];
+        w0[3].s3 = tmp0[3];
+        w1[0].s3 = tmp1[0];
+        w1[1].s3 = tmp1[1];
+        w1[2].s3 = tmp1[2];
+        w1[3].s3 = tmp1[3];
+        out_len.s3 = tmp_len;
+        break;
+      #endif
+
+      #if VECT_SIZE >= 8
+      case 4:
+        w0[0].s4 = tmp0[0];
+        w0[1].s4 = tmp0[1];
+        w0[2].s4 = tmp0[2];
+        w0[3].s4 = tmp0[3];
+        w1[0].s4 = tmp1[0];
+        w1[1].s4 = tmp1[1];
+        w1[2].s4 = tmp1[2];
+        w1[3].s4 = tmp1[3];
+        out_len.s4 = tmp_len;
+        break;
+
+      case 5:
+        w0[0].s5 = tmp0[0];
+        w0[1].s5 = tmp0[1];
+        w0[2].s5 = tmp0[2];
+        w0[3].s5 = tmp0[3];
+        w1[0].s5 = tmp1[0];
+        w1[1].s5 = tmp1[1];
+        w1[2].s5 = tmp1[2];
+        w1[3].s5 = tmp1[3];
+        out_len.s5 = tmp_len;
+        break;
+
+      case 6:
+        w0[0].s6 = tmp0[0];
+        w0[1].s6 = tmp0[1];
+        w0[2].s6 = tmp0[2];
+        w0[3].s6 = tmp0[3];
+        w1[0].s6 = tmp1[0];
+        w1[1].s6 = tmp1[1];
+        w1[2].s6 = tmp1[2];
+        w1[3].s6 = tmp1[3];
+        out_len.s6 = tmp_len;
+        break;
+
+      case 7:
+        w0[0].s7 = tmp0[0];
+        w0[1].s7 = tmp0[1];
+        w0[2].s7 = tmp0[2];
+        w0[3].s7 = tmp0[3];
+        w1[0].s7 = tmp1[0];
+        w1[1].s7 = tmp1[1];
+        w1[2].s7 = tmp1[2];
+        w1[3].s7 = tmp1[3];
+        out_len.s7 = tmp_len;
+        break;
+      #endif
+
+      #if VECT_SIZE >= 16
+      case 8:
+        w0[0].s8 = tmp0[0];
+        w0[1].s8 = tmp0[1];
+        w0[2].s8 = tmp0[2];
+        w0[3].s8 = tmp0[3];
+        w1[0].s8 = tmp1[0];
+        w1[1].s8 = tmp1[1];
+        w1[2].s8 = tmp1[2];
+        w1[3].s8 = tmp1[3];
+        out_len.s8 = tmp_len;
+        break;
+
+      case 9:
+        w0[0].s9 = tmp0[0];
+        w0[1].s9 = tmp0[1];
+        w0[2].s9 = tmp0[2];
+        w0[3].s9 = tmp0[3];
+        w1[0].s9 = tmp1[0];
+        w1[1].s9 = tmp1[1];
+        w1[2].s9 = tmp1[2];
+        w1[3].s9 = tmp1[3];
+        out_len.s9 = tmp_len;
+        break;
+
+      case 10:
+        w0[0].sa = tmp0[0];
+        w0[1].sa = tmp0[1];
+        w0[2].sa = tmp0[2];
+        w0[3].sa = tmp0[3];
+        w1[0].sa = tmp1[0];
+        w1[1].sa = tmp1[1];
+        w1[2].sa = tmp1[2];
+        w1[3].sa = tmp1[3];
+        out_len.sa = tmp_len;
+        break;
+
+      case 11:
+        w0[0].sb = tmp0[0];
+        w0[1].sb = tmp0[1];
+        w0[2].sb = tmp0[2];
+        w0[3].sb = tmp0[3];
+        w1[0].sb = tmp1[0];
+        w1[1].sb = tmp1[1];
+        w1[2].sb = tmp1[2];
+        w1[3].sb = tmp1[3];
+        out_len.sb = tmp_len;
+        break;
+
+      case 12:
+        w0[0].sc = tmp0[0];
+        w0[1].sc = tmp0[1];
+        w0[2].sc = tmp0[2];
+        w0[3].sc = tmp0[3];
+        w1[0].sc = tmp1[0];
+        w1[1].sc = tmp1[1];
+        w1[2].sc = tmp1[2];
+        w1[3].sc = tmp1[3];
+        out_len.sc = tmp_len;
+        break;
+
+      case 13:
+        w0[0].sd = tmp0[0];
+        w0[1].sd = tmp0[1];
+        w0[2].sd = tmp0[2];
+        w0[3].sd = tmp0[3];
+        w1[0].sd = tmp1[0];
+        w1[1].sd = tmp1[1];
+        w1[2].sd = tmp1[2];
+        w1[3].sd = tmp1[3];
+        out_len.sd = tmp_len;
+        break;
+
+      case 14:
+        w0[0].se = tmp0[0];
+        w0[1].se = tmp0[1];
+        w0[2].se = tmp0[2];
+        w0[3].se = tmp0[3];
+        w1[0].se = tmp1[0];
+        w1[1].se = tmp1[1];
+        w1[2].se = tmp1[2];
+        w1[3].se = tmp1[3];
+        out_len.se = tmp_len;
+        break;
+
+      case 15:
+        w0[0].sf = tmp0[0];
+        w0[1].sf = tmp0[1];
+        w0[2].sf = tmp0[2];
+        w0[3].sf = tmp0[3];
+        w1[0].sf = tmp1[0];
+        w1[1].sf = tmp1[1];
+        w1[2].sf = tmp1[2];
+        w1[3].sf = tmp1[3];
+        out_len.sf = tmp_len;
+        break;
+      #endif
+    }
+  }
+
+  return out_len;
+
+  #endif
+}

diff --git a/OpenCL/inc_rp.h b/OpenCL/inc_rp.h
new file mode 100644 (file)
index 0000000..17849f2
--- /dev/null
+++ b/OpenCL/inc_rp.h
@@ -0,0 +1,57 @@
+/**
+ * Author......: Jens Steube <jens.steube@gmail.com>
+ * License.....: MIT
+ */
+
+#define RULE_OP_MANGLE_NOOP             ':'
+#define RULE_OP_MANGLE_LREST            'l'
+#define RULE_OP_MANGLE_UREST            'u'
+#define RULE_OP_MANGLE_LREST_UFIRST     'c'
+#define RULE_OP_MANGLE_UREST_LFIRST     'C'
+#define RULE_OP_MANGLE_TREST            't'
+#define RULE_OP_MANGLE_TOGGLE_AT        'T'
+#define RULE_OP_MANGLE_REVERSE          'r'
+#define RULE_OP_MANGLE_DUPEWORD         'd'
+#define RULE_OP_MANGLE_DUPEWORD_TIMES   'p'
+#define RULE_OP_MANGLE_REFLECT          'f'
+#define RULE_OP_MANGLE_ROTATE_LEFT      '{'
+#define RULE_OP_MANGLE_ROTATE_RIGHT     '}'
+#define RULE_OP_MANGLE_APPEND           '$'
+#define RULE_OP_MANGLE_PREPEND          '^'
+#define RULE_OP_MANGLE_DELETE_FIRST     '['
+#define RULE_OP_MANGLE_DELETE_LAST      ']'
+#define RULE_OP_MANGLE_DELETE_AT        'D'
+#define RULE_OP_MANGLE_EXTRACT          'x'
+#define RULE_OP_MANGLE_OMIT             'O'
+#define RULE_OP_MANGLE_INSERT           'i'
+#define RULE_OP_MANGLE_OVERSTRIKE       'o'
+#define RULE_OP_MANGLE_TRUNCATE_AT      '\''
+#define RULE_OP_MANGLE_REPLACE          's'
+#define RULE_OP_MANGLE_PURGECHAR        '@'
+#define RULE_OP_MANGLE_TOGGLECASE_REC   'a'
+#define RULE_OP_MANGLE_DUPECHAR_FIRST   'z'
+#define RULE_OP_MANGLE_DUPECHAR_LAST    'Z'
+#define RULE_OP_MANGLE_DUPECHAR_ALL     'q'
+
+#define RULE_OP_REJECT_LESS             '<'
+#define RULE_OP_REJECT_GREATER          '>'
+#define RULE_OP_REJECT_CONTAIN          '!'
+#define RULE_OP_REJECT_NOT_CONTAIN      '/'
+#define RULE_OP_REJECT_EQUAL_FIRST      '('
+#define RULE_OP_REJECT_EQUAL_LAST       ')'
+#define RULE_OP_REJECT_EQUAL_AT         '='
+#define RULE_OP_REJECT_CONTAINS         '%'
+
+/* hashcat only */
+#define RULE_OP_MANGLE_SWITCH_FIRST     'k'
+#define RULE_OP_MANGLE_SWITCH_LAST      'K'
+#define RULE_OP_MANGLE_SWITCH_AT        '*'
+#define RULE_OP_MANGLE_CHR_SHIFTL       'L'
+#define RULE_OP_MANGLE_CHR_SHIFTR       'R'
+#define RULE_OP_MANGLE_CHR_INCR         '+'
+#define RULE_OP_MANGLE_CHR_DECR         '-'
+#define RULE_OP_MANGLE_REPLACE_NP1      '.'
+#define RULE_OP_MANGLE_REPLACE_NM1      ','
+#define RULE_OP_MANGLE_DUPEBLOCK_FIRST  'y'
+#define RULE_OP_MANGLE_DUPEBLOCK_LAST   'Y'
+#define RULE_OP_MANGLE_TITLE            'E'

diff --git a/OpenCL/inc_simd.cl b/OpenCL/inc_simd.cl
new file mode 100644 (file)
index 0000000..882e320
--- /dev/null
+++ b/OpenCL/inc_simd.cl
@@ -0,0 +1,1164 @@
+/**
+ * Authors.....: Jens Steube <jens.steube@gmail.com>
+ *               magnum <john.magnum@hushmail.com>
+ *
+ * License.....: MIT
+ */
+
+// vliw1
+
+#if VECT_SIZE == 1
+
+#define MATCHES_ONE_VV(a,b) ((a) == (b))
+#define MATCHES_ONE_VS(a,b) ((a) == (b))
+
+#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  if (((h0) == search[0]) && ((h1) == search[1]) && ((h2) == search[2]) && ((h3) == search[3]))             \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)                                                    \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos);                       \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  const u32 digest_tp0[4] = { h0, h1, h2, h3 };                                                             \
+                                                                                                            \
+  if (check (digest_tp0,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)                                                  \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos);            \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#endif
+
+// vliw2
+
+#define vector_accessible(p,c,e) (((p) + (e)) < (c))
+
+#if VECT_SIZE == 2
+
+#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1))
+#define MATCHES_ONE_VS(a,b) (((a).s0 == (b)   ) || ((a).s1 == (b)   ))
+
+#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 };                                                 \
+  const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 };                                                 \
+                                                                                                            \
+  if (check (digest_tp0,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp1,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#endif
+
+// vliw4
+
+#if VECT_SIZE == 4
+
+#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3))
+#define MATCHES_ONE_VS(a,b) (((a).s0 == (b)   ) || ((a).s1 == (b)   ) || ((a).s2 == (b)   ) || ((a).s3 == (b)   ))
+
+#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 };                                                 \
+  const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 };                                                 \
+  const u32 digest_tp2[4] = { h0.s2, h1.s2, h2.s2, h3.s2 };                                                 \
+  const u32 digest_tp3[4] = { h0.s3, h1.s3, h2.s3, h3.s3 };                                                 \
+                                                                                                            \
+  if (check (digest_tp0,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp1,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp2,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp2, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp3,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp3, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#endif
+
+// vliw8
+
+#if VECT_SIZE == 8
+
+#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3) || ((a).s4 == (b).s4) || ((a).s5 == (b).s5) || ((a).s6 == (b).s6) || ((a).s7 == (b).s7))
+#define MATCHES_ONE_VS(a,b) (((a).s0 == (b)   ) || ((a).s1 == (b)   ) || ((a).s2 == (b)   ) || ((a).s3 == (b)   ) || ((a).s4 == (b)   ) || ((a).s5 == (b)   ) || ((a).s6 == (b)   ) || ((a).s7 == (b)   ))
+
+#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+  if (((h0).s4 == search[0]) && ((h1).s4 == search[1]) && ((h2).s4 == search[2]) && ((h3).s4 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 4);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s5 == search[0]) && ((h1).s5 == search[1]) && ((h2).s5 == search[2]) && ((h3).s5 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 5);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s6 == search[0]) && ((h1).s6 == search[1]) && ((h2).s6 == search[2]) && ((h3).s6 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 6);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s7 == search[0]) && ((h1).s7 == search[1]) && ((h2).s7 == search[2]) && ((h3).s7 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 7);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 };                                                 \
+  const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 };                                                 \
+  const u32 digest_tp2[4] = { h0.s2, h1.s2, h2.s2, h3.s2 };                                                 \
+  const u32 digest_tp3[4] = { h0.s3, h1.s3, h2.s3, h3.s3 };                                                 \
+  const u32 digest_tp4[4] = { h0.s4, h1.s4, h2.s4, h3.s4 };                                                 \
+  const u32 digest_tp5[4] = { h0.s5, h1.s5, h2.s5, h3.s5 };                                                 \
+  const u32 digest_tp6[4] = { h0.s6, h1.s6, h2.s6, h3.s6 };                                                 \
+  const u32 digest_tp7[4] = { h0.s7, h1.s7, h2.s7, h3.s7 };                                                 \
+                                                                                                            \
+  if (check (digest_tp0,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp1,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp2,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp2, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp3,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp3, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+  if (check (digest_tp4,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp4, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 4);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp5,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp5, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 5);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp6,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp6, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 6);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp7,                                                                                    \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp7, digests_cnt, &digests_buf[digests_offset]);                     \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 7);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#endif
+
+// vliw16
+
+#if VECT_SIZE == 16
+
+#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3) || ((a).s4 == (b).s4) || ((a).s5 == (b).s5) || ((a).s6 == (b).s6) || ((a).s7 == (b).s7) || ((a).s8 == (b).s8) || ((a).s9 == (b).s9) || ((a).sa == (b).sa) || ((a).sb == (b).sb) || ((a).sc == (b).sc) || ((a).sd == (b).sd) || ((a).se == (b).se) || ((a).sf == (b).sf))
+#define MATCHES_ONE_VS(a,b) (((a).s0 == (b)   ) || ((a).s1 == (b)   ) || ((a).s2 == (b)   ) || ((a).s3 == (b)   ) || ((a).s4 == (b)   ) || ((a).s5 == (b)   ) || ((a).s6 == (b)   ) || ((a).s7 == (b)   ) || ((a).s8 == (b)   ) || ((a).s9 == (b)   ) || ((a).sa == (b)   ) || ((a).sb == (b)   ) || ((a).sc == (b)   ) || ((a).sd == (b)   ) || ((a).se == (b)   ) || ((a).sf == (b)   ))
+
+#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+  if (((h0).s4 == search[0]) && ((h1).s4 == search[1]) && ((h2).s4 == search[2]) && ((h3).s4 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 4);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s5 == search[0]) && ((h1).s5 == search[1]) && ((h2).s5 == search[2]) && ((h3).s5 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 5);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s6 == search[0]) && ((h1).s6 == search[1]) && ((h2).s6 == search[2]) && ((h3).s6 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 6);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s7 == search[0]) && ((h1).s7 == search[1]) && ((h2).s7 == search[2]) && ((h3).s7 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 7);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s8 == search[0]) && ((h1).s8 == search[1]) && ((h2).s8 == search[2]) && ((h3).s8 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 8) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 8);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).s9 == search[0]) && ((h1).s9 == search[1]) && ((h2).s9 == search[2]) && ((h3).s9 == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 9) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 9);                   \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).sa == search[0]) && ((h1).sa == search[1]) && ((h2).sa == search[2]) && ((h3).sa == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 10) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 10);                  \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).sb == search[0]) && ((h1).sb == search[1]) && ((h2).sb == search[2]) && ((h3).sb == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 11) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 11);                  \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).sc == search[0]) && ((h1).sc == search[1]) && ((h2).sc == search[2]) && ((h3).sc == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 12) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 12);                  \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).sd == search[0]) && ((h1).sd == search[1]) && ((h2).sd == search[2]) && ((h3).sd == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 13) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 13);                  \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).se == search[0]) && ((h1).se == search[1]) && ((h2).se == search[2]) && ((h3).se == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 14) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 14);                  \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (((h0).sf == search[0]) && ((h1).sf == search[1]) && ((h2).sf == search[2]) && ((h3).sf == search[3])) \
+  {                                                                                                         \
+    const u32 final_hash_pos = digests_offset + 0;                                                          \
+                                                                                                            \
+    if (vector_accessible (il_pos, il_cnt, 15) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
+    {                                                                                                       \
+      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 15);                  \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
+{                                                                                                           \
+  const u32 digest_tp00[4] = { h0.s0, h1.s0, h2.s0, h3.s0 };                                                \
+  const u32 digest_tp01[4] = { h0.s1, h1.s1, h2.s1, h3.s1 };                                                \
+  const u32 digest_tp02[4] = { h0.s2, h1.s2, h2.s2, h3.s2 };                                                \
+  const u32 digest_tp03[4] = { h0.s3, h1.s3, h2.s3, h3.s3 };                                                \
+  const u32 digest_tp04[4] = { h0.s4, h1.s4, h2.s4, h3.s4 };                                                \
+  const u32 digest_tp05[4] = { h0.s5, h1.s5, h2.s5, h3.s5 };                                                \
+  const u32 digest_tp06[4] = { h0.s6, h1.s6, h2.s6, h3.s6 };                                                \
+  const u32 digest_tp07[4] = { h0.s7, h1.s7, h2.s7, h3.s7 };                                                \
+  const u32 digest_tp08[4] = { h0.s8, h1.s8, h2.s8, h3.s8 };                                                \
+  const u32 digest_tp09[4] = { h0.s9, h1.s9, h2.s9, h3.s9 };                                                \
+  const u32 digest_tp10[4] = { h0.sa, h1.sa, h2.sa, h3.sa };                                                \
+  const u32 digest_tp11[4] = { h0.sb, h1.sb, h2.sb, h3.sb };                                                \
+  const u32 digest_tp12[4] = { h0.sc, h1.sc, h2.sc, h3.sc };                                                \
+  const u32 digest_tp13[4] = { h0.sd, h1.sd, h2.sd, h3.sd };                                                \
+  const u32 digest_tp14[4] = { h0.se, h1.se, h2.se, h3.se };                                                \
+  const u32 digest_tp15[4] = { h0.sf, h1.sf, h2.sf, h3.sf };                                                \
+                                                                                                            \
+  if (check (digest_tp00,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp00, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp01,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp01, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp02,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp02, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp03,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp03, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp04,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp04, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 4);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp05,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp05, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 5);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp06,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp06, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 6);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp07,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp07, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 7);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp08,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp08, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 8) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 8);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp09,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp09, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 9) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 9);        \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp10,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp10, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 10) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 10);       \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp11,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp11, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 11) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 11);       \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp12,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp12, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 12) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 12);       \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp13,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp13, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 13) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 13);       \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp14,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp14, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 14) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 14);       \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+                                                                                                            \
+  if (check (digest_tp15,                                                                                   \
+             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
+             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
+             bitmap_mask,                                                                                   \
+             bitmap_shift1,                                                                                 \
+             bitmap_shift2))                                                                                \
+  {                                                                                                         \
+    int digest_pos = find_hash (digest_tp15, digests_cnt, &digests_buf[digests_offset]);                    \
+                                                                                                            \
+    if (digest_pos != -1)                                                                                   \
+    {                                                                                                       \
+      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
+                                                                                                            \
+      if (vector_accessible (il_pos, il_cnt, 15) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
+      {                                                                                                     \
+        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 15);       \
+      }                                                                                                     \
+    }                                                                                                       \
+  }                                                                                                         \
+}
+
+#endif
+
+#define MATCHES_NONE_VV(a,b) !(MATCHES_ONE_VV ((a), (b)))
+#define MATCHES_NONE_VS(a,b) !(MATCHES_ONE_VS ((a), (b)))
+
+// attack-mode 0
+
+inline u32x ix_create_bft (__global bf_t *bfs_buf, const u32 il_pos)
+{
+  #if   VECT_SIZE == 1
+  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i);
+  #elif VECT_SIZE == 2
+  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i);
+  #elif VECT_SIZE == 4
+  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i);
+  #elif VECT_SIZE == 8
+  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i, bfs_buf[il_pos + 4].i, bfs_buf[il_pos + 5].i, bfs_buf[il_pos + 6].i, bfs_buf[il_pos + 7].i);
+  #elif VECT_SIZE == 16
+  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i, bfs_buf[il_pos + 4].i, bfs_buf[il_pos + 5].i, bfs_buf[il_pos + 6].i, bfs_buf[il_pos + 7].i, bfs_buf[il_pos + 8].i, bfs_buf[il_pos + 9].i, bfs_buf[il_pos + 10].i, bfs_buf[il_pos + 11].i, bfs_buf[il_pos + 12].i, bfs_buf[il_pos + 13].i, bfs_buf[il_pos + 14].i, bfs_buf[il_pos + 15].i);
+  #endif
+
+  return ix;
+}
+
+// attack-mode 1
+
+inline u32x pwlenx_create_combt (__global comb_t *combs_buf, const u32 il_pos)
+{
+  #if   VECT_SIZE == 1
+  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len);
+  #elif VECT_SIZE == 2
+  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len);
+  #elif VECT_SIZE == 4
+  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len);
+  #elif VECT_SIZE == 8
+  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len, combs_buf[il_pos + 4].pw_len, combs_buf[il_pos + 5].pw_len, combs_buf[il_pos + 6].pw_len, combs_buf[il_pos + 7].pw_len);
+  #elif VECT_SIZE == 16
+  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len, combs_buf[il_pos + 4].pw_len, combs_buf[il_pos + 5].pw_len, combs_buf[il_pos + 6].pw_len, combs_buf[il_pos + 7].pw_len, combs_buf[il_pos + 8].pw_len, combs_buf[il_pos + 9].pw_len, combs_buf[il_pos + 10].pw_len, combs_buf[il_pos + 11].pw_len, combs_buf[il_pos + 12].pw_len, combs_buf[il_pos + 13].pw_len, combs_buf[il_pos + 14].pw_len, combs_buf[il_pos + 15].pw_len);
+  #endif
+
+  return pw_lenx;
+}
+
+inline u32x ix_create_combt (__global comb_t *combs_buf, const u32 il_pos, const int idx)
+{
+  #if   VECT_SIZE == 1
+  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx]);
+  #elif VECT_SIZE == 2
+  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx]);
+  #elif VECT_SIZE == 4
+  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx]);
+  #elif VECT_SIZE == 8
+  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx], combs_buf[il_pos + 4].i[idx], combs_buf[il_pos + 5].i[idx], combs_buf[il_pos + 6].i[idx], combs_buf[il_pos + 7].i[idx]);
+  #elif VECT_SIZE == 16
+  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx], combs_buf[il_pos + 4].i[idx], combs_buf[il_pos + 5].i[idx], combs_buf[il_pos + 6].i[idx], combs_buf[il_pos + 7].i[idx], combs_buf[il_pos + 8].i[idx], combs_buf[il_pos + 9].i[idx], combs_buf[il_pos + 10].i[idx], combs_buf[il_pos + 11].i[idx], combs_buf[il_pos + 12].i[idx], combs_buf[il_pos + 13].i[idx], combs_buf[il_pos + 14].i[idx], combs_buf[il_pos + 15].i[idx]);
+  #endif
+
+  return ix;
+}
+
+#if   VECT_SIZE == 1
+#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) *  1) + 0].var[(idx)])
+#elif VECT_SIZE == 2
+#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) *  2) + 0].var[(idx)], (arr)[((gid) *  2) + 1].var[(idx)])
+#elif VECT_SIZE == 4
+#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) *  4) + 0].var[(idx)], (arr)[((gid) *  4) + 1].var[(idx)], (arr)[((gid) *  4) + 2].var[(idx)], (arr)[((gid) *  4) + 3].var[(idx)])
+#elif VECT_SIZE == 8
+#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) *  8) + 0].var[(idx)], (arr)[((gid) *  8) + 1].var[(idx)], (arr)[((gid) *  8) + 2].var[(idx)], (arr)[((gid) *  8) + 3].var[(idx)], (arr)[((gid) *  8) + 4].var[(idx)], (arr)[((gid) *  8) + 5].var[(idx)], (arr)[((gid) *  8) + 6].var[(idx)], (arr)[((gid) *  8) + 7].var[(idx)])
+#elif VECT_SIZE == 16
+#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 16) + 0].var[(idx)], (arr)[((gid) * 16) + 1].var[(idx)], (arr)[((gid) * 16) + 2].var[(idx)], (arr)[((gid) * 16) + 3].var[(idx)], (arr)[((gid) * 16) + 4].var[(idx)], (arr)[((gid) * 16) + 5].var[(idx)], (arr)[((gid) * 16) + 6].var[(idx)], (arr)[((gid) * 16) + 7].var[(idx)], (arr)[((gid) * 16) + 8].var[(idx)], (arr)[((gid) * 16) + 9].var[(idx)], (arr)[((gid) * 16) + 10].var[(idx)], (arr)[((gid) * 16) + 11].var[(idx)], (arr)[((gid) * 16) + 12].var[(idx)], (arr)[((gid) * 16) + 13].var[(idx)], (arr)[((gid) * 16) + 14].var[(idx)], (arr)[((gid) * 16) + 15].var[(idx)])
+#endif
+
+#if   VECT_SIZE == 1
+#define packvf(arr,var,gid) (u32x) ((arr)[((gid) *  1) + 0].var)
+#elif VECT_SIZE == 2
+#define packvf(arr,var,gid) (u32x) ((arr)[((gid) *  2) + 0].var, (arr)[((gid) *  2) + 1].var)
+#elif VECT_SIZE == 4
+#define packvf(arr,var,gid) (u32x) ((arr)[((gid) *  4) + 0].var, (arr)[((gid) *  4) + 1].var, (arr)[((gid) *  4) + 2].var, (arr)[((gid) *  4) + 3].var)
+#elif VECT_SIZE == 8
+#define packvf(arr,var,gid) (u32x) ((arr)[((gid) *  8) + 0].var, (arr)[((gid) *  8) + 1].var, (arr)[((gid) *  8) + 2].var, (arr)[((gid) *  8) + 3].var, (arr)[((gid) *  8) + 4].var, (arr)[((gid) *  8) + 5].var, (arr)[((gid) *  8) + 6].var, (arr)[((gid) *  8) + 7].var)
+#elif VECT_SIZE == 16
+#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 16) + 0].var, (arr)[((gid) * 16) + 1].var, (arr)[((gid) * 16) + 2].var, (arr)[((gid) * 16) + 3].var, (arr)[((gid) * 16) + 4].var, (arr)[((gid) * 16) + 5].var, (arr)[((gid) * 16) + 6].var, (arr)[((gid) * 16) + 7].var, (arr)[((gid) * 16) + 8].var, (arr)[((gid) * 16) + 9].var, (arr)[((gid) * 16) + 10].var, (arr)[((gid) * 16) + 11].var, (arr)[((gid) * 16) + 12].var, (arr)[((gid) * 16) + 13].var, (arr)[((gid) * 16) + 14].var, (arr)[((gid) * 16) + 15].var)
+#endif
+
+#if   VECT_SIZE == 1
+#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) *  1) + 0].var[(idx)])
+#elif VECT_SIZE == 2
+#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) *  2) + 0].var[(idx)], (arr)[((gid) *  2) + 1].var[(idx)])
+#elif VECT_SIZE == 4
+#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) *  4) + 0].var[(idx)], (arr)[((gid) *  4) + 1].var[(idx)], (arr)[((gid) *  4) + 2].var[(idx)], (arr)[((gid) *  4) + 3].var[(idx)])
+#elif VECT_SIZE == 8
+#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) *  8) + 0].var[(idx)], (arr)[((gid) *  8) + 1].var[(idx)], (arr)[((gid) *  8) + 2].var[(idx)], (arr)[((gid) *  8) + 3].var[(idx)], (arr)[((gid) *  8) + 4].var[(idx)], (arr)[((gid) *  8) + 5].var[(idx)], (arr)[((gid) *  8) + 6].var[(idx)], (arr)[((gid) *  8) + 7].var[(idx)])
+#elif VECT_SIZE == 16
+#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 16) + 0].var[(idx)], (arr)[((gid) * 16) + 1].var[(idx)], (arr)[((gid) * 16) + 2].var[(idx)], (arr)[((gid) * 16) + 3].var[(idx)], (arr)[((gid) * 16) + 4].var[(idx)], (arr)[((gid) * 16) + 5].var[(idx)], (arr)[((gid) * 16) + 6].var[(idx)], (arr)[((gid) * 16) + 7].var[(idx)], (arr)[((gid) * 16) + 8].var[(idx)], (arr)[((gid) * 16) + 9].var[(idx)], (arr)[((gid) * 16) + 10].var[(idx)], (arr)[((gid) * 16) + 11].var[(idx)], (arr)[((gid) * 16) + 12].var[(idx)], (arr)[((gid) * 16) + 13].var[(idx)], (arr)[((gid) * 16) + 14].var[(idx)], (arr)[((gid) * 16) + 15].var[(idx)])
+#endif
+
+#if   VECT_SIZE == 1
+#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) *  1) + 0].var)
+#elif VECT_SIZE == 2
+#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) *  2) + 0].var, (arr)[((gid) *  2) + 1].var)
+#elif VECT_SIZE == 4
+#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) *  4) + 0].var, (arr)[((gid) *  4) + 1].var, (arr)[((gid) *  4) + 2].var, (arr)[((gid) *  4) + 3].var)
+#elif VECT_SIZE == 8
+#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) *  8) + 0].var, (arr)[((gid) *  8) + 1].var, (arr)[((gid) *  8) + 2].var, (arr)[((gid) *  8) + 3].var, (arr)[((gid) *  8) + 4].var, (arr)[((gid) *  8) + 5].var, (arr)[((gid) *  8) + 6].var, (arr)[((gid) *  8) + 7].var)
+#elif VECT_SIZE == 16
+#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 16) + 0].var, (arr)[((gid) * 16) + 1].var, (arr)[((gid) * 16) + 2].var, (arr)[((gid) * 16) + 3].var, (arr)[((gid) * 16) + 4].var, (arr)[((gid) * 16) + 5].var, (arr)[((gid) * 16) + 6].var, (arr)[((gid) * 16) + 7].var, (arr)[((gid) * 16) + 8].var, (arr)[((gid) * 16) + 9].var, (arr)[((gid) * 16) + 10].var, (arr)[((gid) * 16) + 11].var, (arr)[((gid) * 16) + 12].var, (arr)[((gid) * 16) + 13].var, (arr)[((gid) * 16) + 14].var, (arr)[((gid) * 16) + 15].var)
+#endif
+
+#if   VECT_SIZE == 1
+#define unpackv(arr,var,gid,idx,val) (arr)[((gid) *  1) + 0].var[(idx)] = val;
+#elif VECT_SIZE == 2
+#define unpackv(arr,var,gid,idx,val) (arr)[((gid) *  2) + 0].var[(idx)] = val.s0; (arr)[((gid) *  2) + 1].var[(idx)] = val.s1;
+#elif VECT_SIZE == 4
+#define unpackv(arr,var,gid,idx,val) (arr)[((gid) *  4) + 0].var[(idx)] = val.s0; (arr)[((gid) *  4) + 1].var[(idx)] = val.s1; (arr)[((gid) *  4) + 2].var[(idx)] = val.s2; (arr)[((gid) *  4) + 3].var[(idx)] = val.s3;
+#elif VECT_SIZE == 8
+#define unpackv(arr,var,gid,idx,val) (arr)[((gid) *  8) + 0].var[(idx)] = val.s0; (arr)[((gid) *  8) + 1].var[(idx)] = val.s1; (arr)[((gid) *  8) + 2].var[(idx)] = val.s2; (arr)[((gid) *  8) + 3].var[(idx)] = val.s3; (arr)[((gid) *  8) + 4].var[(idx)] = val.s4; (arr)[((gid) *  8) + 5].var[(idx)] = val.s5; (arr)[((gid) *  8) + 6].var[(idx)] = val.s6; (arr)[((gid) *  8) + 7].var[(idx)] = val.s7;
+#elif VECT_SIZE == 16
+#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 16) + 0].var[(idx)] = val.s0; (arr)[((gid) * 16) + 1].var[(idx)] = val.s1; (arr)[((gid) * 16) + 2].var[(idx)] = val.s2; (arr)[((gid) * 16) + 3].var[(idx)] = val.s3; (arr)[((gid) * 16) + 4].var[(idx)] = val.s4; (arr)[((gid) * 16) + 5].var[(idx)] = val.s5; (arr)[((gid) * 16) + 6].var[(idx)] = val.s6; (arr)[((gid) * 16) + 7].var[(idx)] = val.s7; (arr)[((gid) * 16) + 8].var[(idx)] = val.s8; (arr)[((gid) * 16) + 9].var[(idx)] = val.s9; (arr)[((gid) * 16) + 10].var[(idx)] = val.sa; (arr)[((gid) * 16) + 11].var[(idx)] = val.sb; (arr)[((gid) * 16) + 12].var[(idx)] = val.sc; (arr)[((gid) * 16) + 13].var[(idx)] = val.sd; (arr)[((gid) * 16) + 14].var[(idx)] = val.se; (arr)[((gid) * 16) + 15].var[(idx)] = val.sf;
+#endif

diff --git a/OpenCL/inc_types.cl b/OpenCL/inc_types.cl
new file mode 100644 (file)
index 0000000..1d3a7b8
--- /dev/null
+++ b/OpenCL/inc_types.cl
@@ -0,0 +1,1480 @@
+/**
+ * Authors.....: Jens Steube <jens.steube@gmail.com>
+ *               magnum <john.magnum@hushmail.com>
+ *
+ * License.....: MIT
+ */
+
+typedef uchar  u8;
+typedef ushort u16;
+typedef uint   u32;
+typedef ulong  u64;
+
+#ifndef NEW_SIMD_CODE
+#undef  VECT_SIZE
+#define VECT_SIZE 1
+#endif
+
+#define CONCAT(a, b)       a##b
+#define VTYPE(type, width) CONCAT(type, width)
+
+#if VECT_SIZE == 1
+typedef uchar   u8x;
+typedef ushort  u16x;
+typedef uint    u32x;
+typedef ulong   u64x;
+#else
+typedef VTYPE(uchar,  VECT_SIZE)  u8x;
+typedef VTYPE(ushort, VECT_SIZE) u16x;
+typedef VTYPE(uint,   VECT_SIZE) u32x;
+typedef VTYPE(ulong,  VECT_SIZE) u64x;
+#endif
+
+inline u32 l32_from_64_S (u64 a)
+{
+  const u32 r = (u32) (a);
+
+  return r;
+}
+
+inline u32 h32_from_64_S (u64 a)
+{
+  a >>= 32;
+
+  const u32 r = (u32) (a);
+
+  return r;
+}
+
+inline u64 hl32_to_64_S (const u32 a, const u32 b)
+{
+  return as_ulong ((uint2) (b, a));
+}
+
+inline u32x l32_from_64 (u64x a)
+{
+  u32x r;
+
+  #if VECT_SIZE == 1
+  r    = (u32) a;
+  #endif
+
+  #if VECT_SIZE >= 2
+  r.s0 = (u32) a.s0;
+  r.s1 = (u32) a.s1;
+  #endif
+
+  #if VECT_SIZE >= 4
+  r.s2 = (u32) a.s2;
+  r.s3 = (u32) a.s3;
+  #endif
+
+  #if VECT_SIZE >= 8
+  r.s4 = (u32) a.s4;
+  r.s5 = (u32) a.s5;
+  r.s6 = (u32) a.s6;
+  r.s7 = (u32) a.s7;
+  #endif
+
+  #if VECT_SIZE >= 16
+  r.s8 = (u32) a.s8;
+  r.s9 = (u32) a.s9;
+  r.sa = (u32) a.sa;
+  r.sb = (u32) a.sb;
+  r.sc = (u32) a.sc;
+  r.sd = (u32) a.sd;
+  r.se = (u32) a.se;
+  r.sf = (u32) a.sf;
+  #endif
+
+  return r;
+}
+
+inline u32x h32_from_64 (u64x a)
+{
+  a >>= 32;
+
+  u32x r;
+
+  #if VECT_SIZE == 1
+  r    = (u32) a;
+  #endif
+
+  #if VECT_SIZE >= 2
+  r.s0 = (u32) a.s0;
+  r.s1 = (u32) a.s1;
+  #endif
+
+  #if VECT_SIZE >= 4
+  r.s2 = (u32) a.s2;
+  r.s3 = (u32) a.s3;
+  #endif
+
+  #if VECT_SIZE >= 8
+  r.s4 = (u32) a.s4;
+  r.s5 = (u32) a.s5;
+  r.s6 = (u32) a.s6;
+  r.s7 = (u32) a.s7;
+  #endif
+
+  #if VECT_SIZE >= 16
+  r.s8 = (u32) a.s8;
+  r.s9 = (u32) a.s9;
+  r.sa = (u32) a.sa;
+  r.sb = (u32) a.sb;
+  r.sc = (u32) a.sc;
+  r.sd = (u32) a.sd;
+  r.se = (u32) a.se;
+  r.sf = (u32) a.sf;
+  #endif
+
+  return r;
+}
+
+inline u64x hl32_to_64 (const u32x a, const u32x b)
+{
+  u64x r;
+
+  #if VECT_SIZE == 1
+  r    = as_ulong ((uint2) (b,    a));
+  #endif
+
+  #if VECT_SIZE >= 2
+  r.s0 = as_ulong ((uint2) (b.s0, a.s0));
+  r.s1 = as_ulong ((uint2) (b.s1, a.s1));
+  #endif
+
+  #if VECT_SIZE >= 4
+  r.s2 = as_ulong ((uint2) (b.s2, a.s2));
+  r.s3 = as_ulong ((uint2) (b.s3, a.s3));
+  #endif
+
+  #if VECT_SIZE >= 8
+  r.s4 = as_ulong ((uint2) (b.s4, a.s4));
+  r.s5 = as_ulong ((uint2) (b.s5, a.s5));
+  r.s6 = as_ulong ((uint2) (b.s6, a.s6));
+  r.s7 = as_ulong ((uint2) (b.s7, a.s7));
+  #endif
+
+  #if VECT_SIZE >= 16
+  r.s8 = as_ulong ((uint2) (b.s8, a.s8));
+  r.s9 = as_ulong ((uint2) (b.s9, a.s9));
+  r.sa = as_ulong ((uint2) (b.sa, a.sa));
+  r.sb = as_ulong ((uint2) (b.sb, a.sb));
+  r.sc = as_ulong ((uint2) (b.sc, a.sc));
+  r.sd = as_ulong ((uint2) (b.sd, a.sd));
+  r.se = as_ulong ((uint2) (b.se, a.se));
+  r.sf = as_ulong ((uint2) (b.sf, a.sf));
+  #endif
+
+  return r;
+}
+
+#ifdef IS_AMD
+inline u32 swap32_S (const u32 v)
+{
+  return (as_uint (as_uchar4 (v).s3210));
+}
+
+inline u64 swap64_S (const u64 v)
+{
+  return (as_ulong (as_uchar8 (v).s76543210));
+}
+
+inline u32 rotr32_S (const u32 a, const u32 n)
+{
+  return rotate (a, 32 - n);
+}
+
+inline u32 rotl32_S (const u32 a, const u32 n)
+{
+  return rotate (a, n);
+}
+
+inline u64 rotr64_S (const u64 a, const u32 n)
+{
+  const u32 a0 = h32_from_64_S (a);
+  const u32 a1 = l32_from_64_S (a);
+
+  const u32 t0 = (n >= 32) ? amd_bitalign (a0, a1, n - 32) : amd_bitalign (a1, a0, n);
+  const u32 t1 = (n >= 32) ? amd_bitalign (a1, a0, n - 32) : amd_bitalign (a0, a1, n);
+
+  const u64 r = hl32_to_64_S (t0, t1);
+
+  return r;
+}
+
+inline u64 rotl64_S (const u64 a, const u32 n)
+{
+  return rotr64_S (a, 64 - n);
+}
+
+inline u32x swap32 (const u32x v)
+{
+  return ((v >> 24) & 0x000000ff)
+       | ((v >>  8) & 0x0000ff00)
+       | ((v <<  8) & 0x00ff0000)
+       | ((v << 24) & 0xff000000);
+}
+
+inline u64x swap64 (const u64x v)
+{
+  return ((v >> 56) & 0x00000000000000ff)
+       | ((v >> 40) & 0x000000000000ff00)
+       | ((v >> 24) & 0x0000000000ff0000)
+       | ((v >>  8) & 0x00000000ff000000)
+       | ((v <<  8) & 0x000000ff00000000)
+       | ((v << 24) & 0x0000ff0000000000)
+       | ((v << 40) & 0x00ff000000000000)
+       | ((v << 56) & 0xff00000000000000);
+}
+
+inline u32x rotr32 (const u32x a, const u32 n)
+{
+  return rotate (a, 32 - n);
+}
+
+inline u32x rotl32 (const u32x a, const u32 n)
+{
+  return rotate (a, n);
+}
+
+inline u64x rotr64 (const u64x a, const u32 n)
+{
+  const u32x a0 = h32_from_64 (a);
+  const u32x a1 = l32_from_64 (a);
+
+  const u32x t0 = (n >= 32) ? amd_bitalign (a0, a1, n - 32) : amd_bitalign (a1, a0, n);
+  const u32x t1 = (n >= 32) ? amd_bitalign (a1, a0, n - 32) : amd_bitalign (a0, a1, n);
+
+  const u64x r = hl32_to_64 (t0, t1);
+
+  return r;
+}
+
+inline u64x rotl64 (const u64x a, const u32 n)
+{
+  return rotr64 (a, 64 - n);
+}
+
+inline u32x __bfe (const u32x a, const u32x b, const u32x c)
+{
+  return amd_bfe (a, b, c);
+}
+
+inline u32 __bfe_S (const u32 a, const u32 b, const u32 c)
+{
+  return amd_bfe (a, b, c);
+}
+
+inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c)
+{
+  return amd_bytealign (a, b, c);
+}
+#endif
+
+#ifdef IS_NV
+inline u32 swap32_S (const u32 v)
+{
+  u32 r;
+
+  asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(r) : "r"(v));
+
+  return r;
+}
+
+inline u64 swap64_S (const u64 v)
+{
+  u32 il;
+  u32 ir;
+
+  asm ("mov.b64 {%0, %1}, %2;" : "=r"(il), "=r"(ir) : "l"(v));
+
+  u32 tl;
+  u32 tr;
+
+  asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(tl) : "r"(il));
+  asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(tr) : "r"(ir));
+
+  u64 r;
+
+  asm ("mov.b64 %0, {%1, %2};" : "=l"(r) : "r"(tr), "r"(tl));
+
+  return r;
+}
+
+inline u32 rotr32_S (const u32 a, const u32 n)
+{
+  return rotate (a, 32 - n);
+}
+
+inline u32 rotl32_S (const u32 a, const u32 n)
+{
+  return rotate (a, n);
+}
+
+inline u64 rotr64_S (const u64 a, const u32 n)
+{
+  return rotate (a, (u64) 64 - n);
+}
+
+inline u64 rotl64_S (const u64 a, const u32 n)
+{
+  return rotr64_S (a, 64 - n);
+}
+
+inline u32x swap32 (const u32x v)
+{
+  return ((v >> 24) & 0x000000ff)
+       | ((v >>  8) & 0x0000ff00)
+       | ((v <<  8) & 0x00ff0000)
+       | ((v << 24) & 0xff000000);
+}
+
+inline u64x swap64 (const u64x v)
+{
+  return ((v >> 56) & 0x00000000000000ff)
+       | ((v >> 40) & 0x000000000000ff00)
+       | ((v >> 24) & 0x0000000000ff0000)
+       | ((v >>  8) & 0x00000000ff000000)
+       | ((v <<  8) & 0x000000ff00000000)
+       | ((v << 24) & 0x0000ff0000000000)
+       | ((v << 40) & 0x00ff000000000000)
+       | ((v << 56) & 0xff00000000000000);
+}
+
+inline u32x rotr32 (const u32x a, const u32 n)
+{
+  return rotate (a, 32 - n);
+}
+
+inline u32x rotl32 (const u32x a, const u32 n)
+{
+  return rotate (a, n);
+}
+
+inline u64x rotr64 (const u64x a, const u32 n)
+{
+  return rotate (a, (u64) 64 - n);
+}
+
+inline u64x rotl64 (const u64x a, const u32 n)
+{
+  return rotate (a, (u64) n);
+}
+
+inline u32x __byte_perm (const u32x a, const u32x b, const u32x c)
+{
+  u32x r;
+
+  #if VECT_SIZE == 1
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r)    : "r"(a),    "r"(b),    "r"(c)   );
+  #endif
+
+  #if VECT_SIZE >= 2
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(a.s0), "r"(b.s0), "r"(c.s0));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(a.s1), "r"(b.s1), "r"(c.s1));
+  #endif
+
+  #if VECT_SIZE >= 4
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(a.s2), "r"(b.s2), "r"(c.s2));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(a.s3), "r"(b.s3), "r"(c.s3));
+  #endif
+
+  #if VECT_SIZE >= 8
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(a.s4), "r"(b.s4), "r"(c.s4));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(a.s5), "r"(b.s5), "r"(c.s5));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(a.s6), "r"(b.s6), "r"(c.s6));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(a.s7), "r"(b.s7), "r"(c.s7));
+  #endif
+
+  #if VECT_SIZE >= 16
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(a.s8), "r"(b.s8), "r"(c.s8));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(a.s9), "r"(b.s9), "r"(c.s9));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(a.sa), "r"(b.sa), "r"(c.sa));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(a.sb), "r"(b.sb), "r"(c.sb));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(a.sc), "r"(b.sc), "r"(c.sc));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(a.sd), "r"(b.sd), "r"(c.sd));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(a.se), "r"(b.se), "r"(c.se));
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(a.sf), "r"(b.sf), "r"(c.sf));
+  #endif
+
+  return r;
+}
+
+inline u32 __byte_perm_S (const u32 a, const u32 b, const u32 c)
+{
+  u32 r;
+
+  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c));
+
+  return r;
+}
+
+inline u32x __bfe (const u32x a, const u32x b, const u32x c)
+{
+  u32x r;
+
+  #if VECT_SIZE == 1
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r)    : "r"(a),    "r"(b),    "r"(c));
+  #endif
+
+  #if VECT_SIZE >= 2
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(a.s0), "r"(b.s0), "r"(c.s0));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(a.s1), "r"(b.s1), "r"(c.s1));
+  #endif
+
+  #if VECT_SIZE >= 4
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(a.s2), "r"(b.s2), "r"(c.s2));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(a.s3), "r"(b.s3), "r"(c.s3));
+  #endif
+
+  #if VECT_SIZE >= 8
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(a.s4), "r"(b.s4), "r"(c.s4));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(a.s5), "r"(b.s5), "r"(c.s5));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(a.s6), "r"(b.s6), "r"(c.s6));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(a.s7), "r"(b.s7), "r"(c.s7));
+  #endif
+
+  #if VECT_SIZE >= 16
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(a.s8), "r"(b.s8), "r"(c.s8));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(a.s9), "r"(b.s9), "r"(c.s9));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(a.sa), "r"(b.sa), "r"(c.sa));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(a.sb), "r"(b.sb), "r"(c.sb));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(a.sc), "r"(b.sc), "r"(c.sc));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(a.sd), "r"(b.sd), "r"(c.sd));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(a.se), "r"(b.se), "r"(c.se));
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(a.sf), "r"(b.sf), "r"(c.sf));
+  #endif
+
+  return r;
+}
+
+inline u32 __bfe_S (const u32 a, const u32 b, const u32 c)
+{
+  u32 r;
+
+  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c));
+
+  return r;
+}
+
+inline u32x amd_bytealign (const u32x a, const u32x b, const u32x c)
+{
+  u32x r;
+
+  #if CUDA_ARCH >= 350
+
+  #if VECT_SIZE == 1
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r)    : "r"(b),    "r"(a),    "r"((c & 3) * 8));
+  #endif
+
+  #if VECT_SIZE >= 2
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(b.s0), "r"(a.s0), "r"((c.s0 & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(b.s1), "r"(a.s1), "r"((c.s1 & 3) * 8));
+  #endif
+
+  #if VECT_SIZE >= 4
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(b.s2), "r"(a.s2), "r"((c.s2 & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(b.s3), "r"(a.s3), "r"((c.s3 & 3) * 8));
+  #endif
+
+  #if VECT_SIZE >= 8
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(b.s4), "r"(a.s4), "r"((c.s4 & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(b.s5), "r"(a.s5), "r"((c.s5 & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(b.s6), "r"(a.s6), "r"((c.s6 & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(b.s7), "r"(a.s7), "r"((c.s7 & 3) * 8));
+  #endif
+
+  #if VECT_SIZE >= 16
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(b.s8), "r"(a.s8), "r"((c.s8 & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(b.s9), "r"(a.s9), "r"((c.s9 & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(b.sa), "r"(a.sa), "r"((c.sa & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(b.sb), "r"(a.sb), "r"((c.sb & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(b.sc), "r"(a.sc), "r"((c.sc & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(b.sd), "r"(a.sd), "r"((c.sd & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(b.se), "r"(a.se), "r"((c.se & 3) * 8));
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(b.sf), "r"(a.sf), "r"((c.sf & 3) * 8));
+  #endif
+
+  #else
+
+  r = __byte_perm (b, a, ((u32x) (0x76543210) >> ((c & 3) * 4)) & 0xffff);
+
+  #endif
+
+  return r;
+}
+
+inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c)
+{
+  u32 r;
+
+  #if CUDA_ARCH >= 350
+
+  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(b), "r"(a), "r"((c & 3) * 8));
+
+  #else
+
+  r = __byte_perm_S (b, a, (0x76543210 >> ((c & 3) * 4)) & 0xffff);
+
+  #endif
+
+  return r;
+}
+#endif
+
+#ifdef IS_GENERIC
+inline u32 swap32_S (const u32 v)
+{
+  return (as_uint (as_uchar4 (v).s3210));
+}
+
+inline u64 swap64_S (const u64 v)
+{
+  return (as_ulong (as_uchar8 (v).s76543210));
+}
+
+inline u32 rotr32_S (const u32 a, const u32 n)
+{
+  return rotate (a, 32 - n);
+}
+
+inline u32 rotl32_S (const u32 a, const u32 n)
+{
+  return rotate (a, n);
+}
+
+inline u64 rotr64_S (const u64 a, const u32 n)
+{
+  return rotate (a, (u64) 64 - n);
+}
+
+inline u64 rotl64_S (const u64 a, const u32 n)
+{
+  return rotate (a, (u64) n);
+}
+
+inline u32x swap32 (const u32x v)
+{
+  return ((v >> 24) & 0x000000ff)
+       | ((v >>  8) & 0x0000ff00)
+       | ((v <<  8) & 0x00ff0000)
+       | ((v << 24) & 0xff000000);
+}
+
+inline u64x swap64 (const u64x v)
+{
+  return ((v >> 56) & 0x00000000000000ff)
+       | ((v >> 40) & 0x000000000000ff00)
+       | ((v >> 24) & 0x0000000000ff0000)
+       | ((v >>  8) & 0x00000000ff000000)
+       | ((v <<  8) & 0x000000ff00000000)
+       | ((v << 24) & 0x0000ff0000000000)
+       | ((v << 40) & 0x00ff000000000000)
+       | ((v << 56) & 0xff00000000000000);
+}
+
+inline u32x rotr32 (const u32x a, const u32 n)
+{
+  return rotate (a, 32 - n);
+}
+
+inline u32x rotl32 (const u32x a, const u32 n)
+{
+  return rotate (a, n);
+}
+
+inline u64x rotr64 (const u64x a, const u32 n)
+{
+  return rotate (a, (u64) 64 - n);
+}
+
+inline u64x rotl64 (const u64x a, const u32 n)
+{
+  return rotate (a, (u64) n);
+}
+
+inline u32x __bfe (const u32x a, const u32x b, const u32x c)
+{
+  #define BIT(x)      ((u32x) (1u) << (x))
+  #define BIT_MASK(x) (BIT (x) - 1)
+  #define BFE(x,y,z)  (((x) >> (y)) & BIT_MASK (z))
+
+  return BFE (a, b, c);
+
+  #undef BIT
+  #undef BIT_MASK
+  #undef BFE
+}
+
+inline u32 __bfe_S (const u32 a, const u32 b, const u32 c)
+{
+  #define BIT(x)      (1u << (x))
+  #define BIT_MASK(x) (BIT (x) - 1)
+  #define BFE(x,y,z)  (((x) >> (y)) & BIT_MASK (z))
+
+  return BFE (a, b, c);
+
+  #undef BIT
+  #undef BIT_MASK
+  #undef BFE
+}
+
+inline u32x amd_bytealign (const u32x a, const u32x b, const u32 c)
+{
+  #if VECT_SIZE == 1
+  const u64x tmp = ((((u64x) (a)) << 32) | ((u64x) (b))) >> ((c & 3) * 8);
+
+  return (u32x) (tmp);
+  #endif
+
+  #if VECT_SIZE == 2
+  const u64x tmp = ((((u64x) (a.s0, a.s1)) << 32) | ((u64x) (b.s0, b.s1))) >> ((c & 3) * 8);
+
+  return (u32x) (tmp.s0, tmp.s1);
+  #endif
+
+  #if VECT_SIZE == 4
+  const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3))) >> ((c & 3) * 8);
+
+  return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3);
+  #endif
+
+  #if VECT_SIZE == 8
+  const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3, a.s4, a.s5, a.s6, a.s7)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3, b.s4, b.s5, b.s6, b.s7))) >> ((c & 3) * 8);
+
+  return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3, tmp.s4, tmp.s5, tmp.s6, tmp.s7);
+  #endif
+
+  #if VECT_SIZE == 16
+  const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3, a.s4, a.s5, a.s6, a.s7, a.s8, a.s9, a.sa, a.sb, a.sc, a.sd, a.se, a.sf)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3, b.s4, b.s5, b.s6, b.s7, b.s8, b.s9, b.sa, b.sb, b.sc, b.sd, b.se, b.sf))) >> ((c & 3) * 8);
+
+  return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3, tmp.s4, tmp.s5, tmp.s6, tmp.s7, tmp.s8, tmp.s9, tmp.sa, tmp.sb, tmp.sc, tmp.sd, tmp.se, tmp.sf);
+  #endif
+}
+
+inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c)
+{
+  const u64 tmp = ((((u64) a) << 32) | ((u64) b)) >> ((c & 3) * 8);
+
+  return (u32) (tmp);
+}
+
+#endif
+
+typedef struct
+{
+  #if   defined _DES_
+  u32  digest_buf[4];
+  #elif defined _MD4_
+  u32  digest_buf[4];
+  #elif defined _MD5_
+  u32  digest_buf[4];
+  #elif defined _MD5H_
+  u32  digest_buf[4];
+  #elif defined _SHA1_
+  u32  digest_buf[5];
+  #elif defined _BCRYPT_
+  u32  digest_buf[6];
+  #elif defined _SHA256_
+  u32  digest_buf[8];
+  #elif defined _SHA384_
+  u32  digest_buf[16];
+  #elif defined _SHA512_
+  u32  digest_buf[16];
+  #elif defined _KECCAK_
+  u32  digest_buf[50];
+  #elif defined _RIPEMD160_
+  u32  digest_buf[5];
+  #elif defined _WHIRLPOOL_
+  u32  digest_buf[16];
+  #elif defined _GOST_
+  u32  digest_buf[8];
+  #elif defined _GOST2012_256_
+  u32  digest_buf[8];
+  #elif defined _GOST2012_512_
+  u32  digest_buf[16];
+  #elif defined _SAPB_
+  u32  digest_buf[4];
+  #elif defined _SAPG_
+  u32  digest_buf[5];
+  #elif defined _MYSQL323_
+  u32  digest_buf[4];
+  #elif defined _LOTUS5_
+  u32  digest_buf[4];
+  #elif defined _LOTUS6_
+  u32  digest_buf[4];
+  #elif defined _SCRYPT_
+  u32  digest_buf[8];
+  #elif defined _LOTUS8_
+  u32  digest_buf[4];
+  #elif defined _OFFICE2007_
+  u32  digest_buf[4];
+  #elif defined _OFFICE2010_
+  u32  digest_buf[4];
+  #elif defined _OFFICE2013_
+  u32  digest_buf[4];
+  #elif defined _OLDOFFICE01_
+  u32  digest_buf[4];
+  #elif defined _OLDOFFICE34_
+  u32  digest_buf[4];
+  #elif defined _SIPHASH_
+  u32  digest_buf[4];
+  #elif defined _PBKDF2_MD5_
+  u32  digest_buf[32];
+  #elif defined _PBKDF2_SHA1_
+  u32  digest_buf[32];
+  #elif defined _PBKDF2_SHA256_
+  u32  digest_buf[32];
+  #elif defined _PBKDF2_SHA512_
+  u32  digest_buf[32];
+  #elif defined _PDF17L8_
+  u32  digest_buf[8];
+  #elif defined _CRC32_
+  u32  digest_buf[4];
+  #elif defined _SEVEN_ZIP_
+  u32  digest_buf[4];
+  #elif defined _ANDROIDFDE_
+  u32  digest_buf[4];
+  #elif defined _DCC2_
+  u32  digest_buf[4];
+  #elif defined _WPA_
+  u32  digest_buf[4];
+  #elif defined _MD5_SHA1_
+  u32  digest_buf[4];
+  #elif defined _SHA1_MD5_
+  u32  digest_buf[5];
+  #elif defined _NETNTLMV2_
+  u32  digest_buf[4];
+  #elif defined _KRB5PA_
+  u32  digest_buf[4];
+  #elif defined _CLOUDKEY_
+  u32  digest_buf[8];
+  #elif defined _SCRYPT_
+  u32  digest_buf[4];
+  #elif defined _PSAFE2_
+  u32  digest_buf[5];
+  #elif defined _LOTUS8_
+  u32  digest_buf[4];
+  #elif defined _RAR3_
+  u32  digest_buf[4];
+  #elif defined _SHA256_SHA1_
+  u32  digest_buf[8];
+  #elif defined _MS_DRSR_
+  u32  digest_buf[8];
+  #elif defined _ANDROIDFDE_SAMSUNG_
+  u32  digest_buf[8];
+  #elif defined _RAR5_
+  u32  digest_buf[4];
+  #elif defined _KRB5TGS_
+  u32  digest_buf[4];
+  #elif defined _AXCRYPT_
+  u32  digest_buf[4];
+  #elif defined _KEEPASS_
+  u32  digest_buf[4];
+  #elif defined _ZIP2_
+  u32  digest_buf[4];
+  #endif
+
+} digest_t;
+
+typedef struct
+{
+  u32 salt_buf[16];
+  u32 salt_buf_pc[8];
+
+  u32 salt_len;
+  u32 salt_iter;
+  u32 salt_sign[2];
+
+  u32 keccak_mdlen;
+  u32 truecrypt_mdlen;
+
+  u32 digests_cnt;
+  u32 digests_done;
+
+  u32 digests_offset;
+
+  u32 scrypt_N;
+  u32 scrypt_r;
+  u32 scrypt_p;
+  u32 scrypt_tmto;
+  u32 scrypt_phy;
+
+} salt_t;
+
+typedef struct
+{
+  int V;
+  int R;
+  int P;
+
+  int enc_md;
+
+  u32 id_buf[8];
+  u32 u_buf[32];
+  u32 o_buf[32];
+
+  int id_len;
+  int o_len;
+  int u_len;
+
+  u32 rc4key[2];
+  u32 rc4data[2];
+
+} pdf_t;
+
+typedef struct
+{
+  u32 pke[25];
+  u32 eapol[64];
+  int eapol_size;
+  int keyver;
+  u8  orig_mac1[6];
+  u8  orig_mac2[6];
+  u8  orig_nonce1[32];
+  u8  orig_nonce2[32];
+
+} wpa_t;
+
+typedef struct
+{
+  u32 cry_master_buf[64];
+  u32 ckey_buf[64];
+  u32 public_key_buf[64];
+
+  u32 cry_master_len;
+  u32 ckey_len;
+  u32 public_key_len;
+
+} bitcoin_wallet_t;
+
+typedef struct
+{
+  u32 salt_buf[30];
+  u32 salt_len;
+
+  u32 esalt_buf[38];
+  u32 esalt_len;
+
+} sip_t;
+
+typedef struct
+{
+  u32 data[384];
+
+} androidfde_t;
+
+typedef struct
+{
+  u32 nr_buf[16];
+  u32 nr_len;
+
+  u32 msg_buf[128];
+  u32 msg_len;
+
+} ikepsk_t;
+
+typedef struct
+{
+  u32 user_len;
+  u32 domain_len;
+  u32 srvchall_len;
+  u32 clichall_len;
+
+  u32 userdomain_buf[64];
+  u32 chall_buf[256];
+
+} netntlm_t;
+
+typedef struct
+{
+  u32 user[16];
+  u32 realm[16];
+  u32 salt[32];
+  u32 timestamp[16];
+  u32 checksum[4];
+
+} krb5pa_t;
+
+typedef struct
+{
+  u32 account_info[512];
+  u32 checksum[4];
+  u32 edata2[2560];
+  u32 edata2_len;
+
+} krb5tgs_t;
+
+typedef struct
+{
+  u32 salt_buf[16];
+  u32 data_buf[112];
+  u32 keyfile_buf[16];
+  u32 signature;
+
+} tc_t;
+
+typedef struct
+{
+  u32 salt_buf[16];
+
+} pbkdf2_md5_t;
+
+typedef struct
+{
+  u32 salt_buf[16];
+
+} pbkdf2_sha1_t;
+
+typedef struct
+{
+  u32 salt_buf[16];
+
+} pbkdf2_sha256_t;
+
+typedef struct
+{
+  u32 salt_buf[32];
+
+} pbkdf2_sha512_t;
+
+typedef struct
+{
+  u32 salt_buf[128];
+  u32 salt_len;
+
+} rakp_t;
+
+typedef struct
+{
+  u32 data_len;
+  u32 data_buf[512];
+
+} cloudkey_t;
+
+typedef struct
+{
+  u32 encryptedVerifier[4];
+  u32 encryptedVerifierHash[5];
+
+  u32 keySize;
+
+} office2007_t;
+
+typedef struct
+{
+  u32 encryptedVerifier[4];
+  u32 encryptedVerifierHash[8];
+
+} office2010_t;
+
+typedef struct
+{
+  u32 encryptedVerifier[4];
+  u32 encryptedVerifierHash[8];
+
+} office2013_t;
+
+typedef struct
+{
+  u32 version;
+  u32 encryptedVerifier[4];
+  u32 encryptedVerifierHash[4];
+  u32 rc4key[2];
+
+} oldoffice01_t;
+
+typedef struct
+{
+  u32 version;
+  u32 encryptedVerifier[4];
+  u32 encryptedVerifierHash[5];
+  u32 rc4key[2];
+
+} oldoffice34_t;
+
+typedef struct
+{
+  u32 salt_buf[128];
+  u32 salt_len;
+
+  u32 pc_digest[5];
+  u32 pc_offset;
+
+} pstoken_t;
+
+typedef struct
+{
+  u32 type;
+  u32 mode;
+  u32 magic;
+  u32 salt_len;
+  u32 salt_buf[4];
+  u32 verify_bytes;
+  u32 compress_length;
+  u32 data_len;
+  u32 data_buf[2048];
+  u32 auth_len;
+  u32 auth_buf[4];
+
+} zip2_t;
+
+typedef struct
+{
+  u32 version;
+  u32 algorithm;
+
+  /* key-file handling */
+  u32 keyfile_len;
+  u32 keyfile[8];
+
+  u32 final_random_seed[8];
+  u32 transf_random_seed[8];
+  u32 enc_iv[4];
+  u32 contents_hash[8];
+
+  /* specific to version 1 */
+  u32 contents_len;
+  u32 contents[75000];
+
+  /* specific to version 2 */
+  u32 expected_bytes[8];
+
+} keepass_t;
+
+typedef struct
+{
+  u32 digest[4];
+  u32 out[4];
+
+} pdf14_tmp_t;
+
+typedef struct
+{
+  union
+  {
+    u32 dgst32[16];
+    u64 dgst64[8];
+  };
+
+  u32 dgst_len;
+  u32 W_len;
+
+} pdf17l8_tmp_t;
+
+typedef struct
+{
+  u32 digest_buf[4];
+
+} phpass_tmp_t;
+
+typedef struct
+{
+  u32 digest_buf[4];
+
+} md5crypt_tmp_t;
+
+typedef struct
+{
+  u32 alt_result[8];
+
+  u32 p_bytes[4];
+  u32 s_bytes[4];
+
+} sha256crypt_tmp_t;
+
+typedef struct
+{
+  u64 l_alt_result[8];
+
+  u64 l_p_bytes[2];
+  u64 l_s_bytes[2];
+
+} sha512crypt_tmp_t;
+
+typedef struct
+{
+  u32 ipad[5];
+  u32 opad[5];
+
+  u32 dgst[10];
+  u32 out[10];
+
+} wpa_tmp_t;
+
+typedef struct
+{
+  u64 dgst[8];
+
+} bitcoin_wallet_tmp_t;
+
+typedef struct
+{
+  u32 ipad[5];
+  u32 opad[5];
+
+  u32 dgst[5];
+  u32 out[4];
+
+} dcc2_tmp_t;
+
+typedef struct
+{
+  u32 E[18];
+
+  u32 P[18];
+
+  u32 S0[256];
+  u32 S1[256];
+  u32 S2[256];
+  u32 S3[256];
+
+} bcrypt_tmp_t;
+
+typedef struct
+{
+  u32 digest[2];
+
+  u32 P[18];
+
+  u32 S0[256];
+  u32 S1[256];
+  u32 S2[256];
+  u32 S3[256];
+
+} pwsafe2_tmp_t;
+
+typedef struct
+{
+  u32 digest_buf[8];
+
+} pwsafe3_tmp_t;
+
+typedef struct
+{
+  u32 digest_buf[5];
+
+} androidpin_tmp_t;
+
+typedef struct
+{
+  u32 ipad[5];
+  u32 opad[5];
+
+  u32 dgst[10];
+  u32 out[10];
+
+} androidfde_tmp_t;
+
+typedef struct
+{
+  u32 ipad[16];
+  u32 opad[16];
+
+  u32 dgst[64];
+  u32 out[64];
+
+} tc_tmp_t;
+
+typedef struct
+{
+  u64 ipad[8];
+  u64 opad[8];
+
+  u64 dgst[32];
+  u64 out[32];
+
+} tc64_tmp_t;
+
+typedef struct
+{
+  u32 ipad[4];
+  u32 opad[4];
+
+  u32 dgst[32];
+  u32 out[32];
+
+} pbkdf2_md5_tmp_t;
+
+typedef struct
+{
+  u32 ipad[5];
+  u32 opad[5];
+
+  u32 dgst[32];
+  u32 out[32];
+
+} pbkdf2_sha1_tmp_t;
+
+typedef struct
+{
+  u32 ipad[8];
+  u32 opad[8];
+
+  u32 dgst[32];
+  u32 out[32];
+
+} pbkdf2_sha256_tmp_t;
+
+typedef struct
+{
+  u64 ipad[8];
+  u64 opad[8];
+
+  u64 dgst[16];
+  u64 out[16];
+
+} pbkdf2_sha512_tmp_t;
+
+typedef struct
+{
+  u64 out[8];
+
+} ecryptfs_tmp_t;
+
+typedef struct
+{
+  u64 ipad[8];
+  u64 opad[8];
+
+  u64 dgst[16];
+  u64 out[16];
+
+} oraclet_tmp_t;
+
+typedef struct
+{
+  u32 ipad[5];
+  u32 opad[5];
+
+  u32 dgst[5];
+  u32 out[5];
+
+} agilekey_tmp_t;
+
+typedef struct
+{
+  u32 ipad[5];
+  u32 opad[5];
+
+  u32 dgst1[5];
+  u32 out1[5];
+
+  u32 dgst2[5];
+  u32 out2[5];
+
+} mywallet_tmp_t;
+
+typedef struct
+{
+  u32 ipad[5];
+  u32 opad[5];
+
+  u32 dgst[5];
+  u32 out[5];
+
+} sha1aix_tmp_t;
+
+typedef struct
+{
+  u32 ipad[8];
+  u32 opad[8];
+
+  u32 dgst[8];
+  u32 out[8];
+
+} sha256aix_tmp_t;
+
+typedef struct
+{
+  u64 ipad[8];
+  u64 opad[8];
+
+  u64 dgst[8];
+  u64 out[8];
+
+} sha512aix_tmp_t;
+
+typedef struct
+{
+  u32 ipad[8];
+  u32 opad[8];
+
+  u32 dgst[8];
+  u32 out[8];
+
+} lastpass_tmp_t;
+
+typedef struct
+{
+  u64 digest_buf[8];
+
+} drupal7_tmp_t;
+
+typedef struct
+{
+  u32 ipad[5];
+  u32 opad[5];
+
+  u32 dgst[5];
+  u32 out[5];
+
+} lotus8_tmp_t;
+
+typedef struct
+{
+  u32 out[5];
+
+} office2007_tmp_t;
+
+typedef struct
+{
+  u32 out[5];
+
+} office2010_tmp_t;
+
+typedef struct
+{
+  u64 out[8];
+
+} office2013_tmp_t;
+
+typedef struct
+{
+  u32 digest_buf[5];
+
+} saph_sha1_tmp_t;
+
+typedef struct
+{
+  u32 block[16];
+
+  u32 dgst[8];
+
+  u32 block_len;
+  u32 final_len;
+
+} seven_zip_tmp_t;
+
+typedef struct
+{
+  u32 KEK[5];
+
+  u32 lsb[4];
+  u32 cipher[4];
+
+} axcrypt_tmp_t;
+
+typedef struct
+{
+  u32 tmp_digest[8];
+
+} keepass_tmp_t;
+
+typedef struct
+{
+  u32 Kc[16];
+  u32 Kd[16];
+
+  u32 iv[2];
+
+} bsdicrypt_tmp_t;
+
+typedef struct
+{
+  u32 dgst[17][5];
+
+} rar3_tmp_t;
+
+typedef struct
+{
+  u32 user[16];
+
+} cram_md5_t;
+
+typedef struct
+{
+  u32 iv_buf[4];
+  u32 iv_len;
+
+  u32 salt_buf[4];
+  u32 salt_len;
+
+  u32 crc;
+
+  u32 data_buf[96];
+  u32 data_len;
+
+  u32 unpack_size;
+
+} seven_zip_t;
+
+typedef struct
+{
+  u32 key;
+  u64 val;
+
+} hcstat_table_t;
+
+typedef struct
+{
+  u32 cs_buf[0x100];
+  u32 cs_len;
+
+} cs_t;
+
+typedef struct
+{
+  u32 cmds[0x100];
+
+} kernel_rule_t;
+
+typedef struct
+{
+  u32 salt_pos;
+  u32 digest_pos;
+  u32 hash_pos;
+  u32 gidvid;
+  u32 il_pos;
+
+} plain_t;
+
+typedef struct
+{
+  u32 i[16];
+
+  u32 pw_len;
+
+  u32 alignment_placeholder_1;
+  u32 alignment_placeholder_2;
+  u32 alignment_placeholder_3;
+
+} pw_t;
+
+typedef struct
+{
+  u32 i;
+
+} bf_t;
+
+typedef struct
+{
+  u32 i[8];
+
+  u32 pw_len;
+
+} comb_t;
+
+typedef struct
+{
+  u32 b[32];
+
+} bs_word_t;
+
+typedef struct
+{
+  uint4 P[64];
+
+} scrypt_tmp_t;

diff --git a/OpenCL/inc_vendor.cl b/OpenCL/inc_vendor.cl
new file mode 100644 (file)
index 0000000..3adde96
--- /dev/null
+++ b/OpenCL/inc_vendor.cl
@@ -0,0 +1,237 @@
+/**
+ * Author......: Jens Steube <jens.steube@gmail.com>
+ * License.....: MIT
+ */
+
+//fails on intel opencl sdk
+//#pragma OPENCL EXTENSION cl_khr_int64_base_atomics     : enable
+//#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
+
+/**
+ * device type
+ */
+
+#define DEVICE_TYPE_CPU   2
+#define DEVICE_TYPE_GPU   4
+#define DEVICE_TYPE_ACCEL 8
+
+#if   DEVICE_TYPE == DEVICE_TYPE_CPU
+#define IS_CPU
+#elif DEVICE_TYPE == DEVICE_TYPE_GPU
+#define IS_GPU
+#elif DEVICE_TYPE == DEVICE_TYPE_ACCEL
+#define IS_ACCEL
+#endif
+
+/**
+ * vendor specific
+ */
+
+#if   VENDOR_ID == (1 << 0)
+#define IS_AMD
+//#define IS_GENERIC
+#elif VENDOR_ID == (1 << 1)
+#define IS_APPLE
+#define IS_GENERIC
+#elif VENDOR_ID == (1 << 2)
+#define IS_INTEL_BEIGNET
+#define IS_GENERIC
+#elif VENDOR_ID == (1 << 3)
+#define IS_INTEL_SDK
+#define IS_GENERIC
+#elif VENDOR_ID == (1 << 4)
+#define IS_MESA
+#define IS_GENERIC
+#elif VENDOR_ID == (1 << 5)
+#define IS_NV
+//#define IS_GENERIC
+#elif VENDOR_ID == (1 << 6)
+#define IS_POCL
+#define IS_GENERIC
+#else
+#define IS_GENERIC
+#endif
+
+/**
+ * AMD specific
+ */
+
+#ifdef IS_AMD
+#pragma OPENCL EXTENSION cl_amd_media_ops  : enable
+#pragma OPENCL EXTENSION cl_amd_media_ops2 : enable
+#endif
+
+/**
+ * Unrolling is generally enabled, for all device types and hash modes
+ * There's a few exception when it's better not to unroll
+ */
+
+// Some algorithms run into too much register pressure due to loop unrolling
+
+#ifdef IS_NV
+#ifdef IS_GPU
+
+#if KERN_TYPE == 1500
+#undef _unroll
+#endif
+#if KERN_TYPE == 1800
+#undef _unroll
+#endif
+#if KERN_TYPE == 3000
+#undef _unroll
+#endif
+#if KERN_TYPE == 6221
+#undef _unroll
+#endif
+#if KERN_TYPE == 6222
+#undef _unroll
+#endif
+#if KERN_TYPE == 6223
+#undef _unroll
+#endif
+#if KERN_TYPE == 6500
+#undef _unroll
+#endif
+#if KERN_TYPE == 7100
+#undef _unroll
+#endif
+#if KERN_TYPE == 7400
+#undef _unroll
+#endif
+#if KERN_TYPE == 7900
+#undef _unroll
+#endif
+#if KERN_TYPE == 8200
+#undef _unroll
+#endif
+#if KERN_TYPE == 10400
+#undef _unroll
+#endif
+#if KERN_TYPE == 10500
+#undef _unroll
+#endif
+#if KERN_TYPE == 10700
+#undef _unroll
+#endif
+#if KERN_TYPE == 12300
+#undef _unroll
+#endif
+#if KERN_TYPE == 12400
+#undef _unroll
+#endif
+#if KERN_TYPE == 13721
+#undef _unroll
+#endif
+#if KERN_TYPE == 13722
+#undef _unroll
+#endif
+#if KERN_TYPE == 13723
+#undef _unroll
+#endif
+
+#endif
+#endif
+
+#ifdef IS_AMD
+#ifdef IS_GPU
+
+#if KERN_TYPE == 3200
+#undef _unroll
+#endif
+#if KERN_TYPE == 5200
+#undef _unroll
+#endif
+#if KERN_TYPE == 6100
+#undef _unroll
+#endif
+#if KERN_TYPE == 6221
+#undef _unroll
+#endif
+#if KERN_TYPE == 6222
+#undef _unroll
+#endif
+#if KERN_TYPE == 6223
+#undef _unroll
+#endif
+#if KERN_TYPE == 6400
+#undef _unroll
+#endif
+#if KERN_TYPE == 6500
+#undef _unroll
+#endif
+#if KERN_TYPE == 6800
+#undef _unroll
+#endif
+#if KERN_TYPE == 7100
+#undef _unroll
+#endif
+#if KERN_TYPE == 7400
+#undef _unroll
+#endif
+#if KERN_TYPE == 8000
+#undef _unroll
+#endif
+#if KERN_TYPE == 8200
+#undef _unroll
+#endif
+#if KERN_TYPE == 10900
+#undef _unroll
+#endif
+#if KERN_TYPE == 11600
+#undef _unroll
+#endif
+#if KERN_TYPE == 12300
+#undef _unroll
+#endif
+#if KERN_TYPE == 12800
+#undef _unroll
+#endif
+#if KERN_TYPE == 12900
+#undef _unroll
+#endif
+#if KERN_TYPE == 13000
+#undef _unroll
+#endif
+#if KERN_TYPE == 13721
+#undef _unroll
+#endif
+#if KERN_TYPE == 13722
+#undef _unroll
+#endif
+#if KERN_TYPE == 13723
+#undef _unroll
+#endif
+
+#endif
+#endif
+
+// Some algorithms break due to loop unrolling, it's unknown why, probably compiler bugs
+// Can overlap with above cases
+
+#ifdef IS_AMD
+#ifdef IS_GPU
+
+#if KERN_TYPE == 1750
+#undef _unroll
+#endif
+#if KERN_TYPE == 1760
+#undef _unroll
+#endif
+#if KERN_TYPE == 6500
+#undef _unroll
+#endif
+#if KERN_TYPE == 7100
+#undef _unroll
+#endif
+#if KERN_TYPE == 9600
+#undef _unroll
+#endif
+#if KERN_TYPE == 12200
+#undef _unroll
+#endif
+#if KERN_TYPE == 12300
+#undef _unroll
+#endif
+
+#endif
+#endif

diff --git a/OpenCL/kernel_aes256.c b/OpenCL/kernel_aes256.c
deleted file mode 100644 (file)
index b3b6107..0000000
--- a/OpenCL/kernel_aes256.c
+++ /dev/null
@@ -1,1048 +0,0 @@
-/**
- * Author......: Jens Steube <jens.steube@gmail.com>
- * License.....: MIT
- */
-
-__constant u32 te0[256] =
-{
-  0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d,
-  0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554,
-  0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d,
-  0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a,
-  0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87,
-  0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b,
-  0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea,
-  0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b,
-  0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a,
-  0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f,
-  0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108,
-  0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f,
-  0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e,
-  0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5,
-  0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d,
-  0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f,
-  0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e,
-  0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb,
-  0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce,
-  0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497,
-  0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c,
-  0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed,
-  0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b,
-  0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a,
-  0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16,
-  0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594,
-  0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81,
-  0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3,
-  0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a,
-  0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504,
-  0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163,
-  0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d,
-  0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f,
-  0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739,
-  0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47,
-  0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395,
-  0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f,
-  0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883,
-  0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c,
-  0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76,
-  0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e,
-  0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4,
-  0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6,
-  0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b,
-  0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7,
-  0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0,
-  0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25,
-  0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818,
-  0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72,
-  0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651,
-  0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21,
-  0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85,
-  0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa,
-  0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12,
-  0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0,
-  0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9,
-  0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133,
-  0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7,
-  0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920,
-  0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a,
-  0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17,
-  0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8,
-  0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11,
-  0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a,
-};
-
-__constant u32 te1[256] =
-{
-  0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b,
-  0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5,
-  0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b,
-  0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676,
-  0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d,
-  0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0,
-  0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf,
-  0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0,
-  0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626,
-  0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc,
-  0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1,
-  0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515,
-  0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3,
-  0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a,
-  0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2,
-  0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575,
-  0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a,
-  0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0,
-  0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3,
-  0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484,
-  0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded,
-  0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b,
-  0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939,
-  0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf,
-  0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb,
-  0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585,
-  0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f,
-  0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8,
-  0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f,
-  0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5,
-  0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121,
-  0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2,
-  0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec,
-  0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717,
-  0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d,
-  0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373,
-  0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc,
-  0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888,
-  0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414,
-  0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb,
-  0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a,
-  0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c,
-  0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262,
-  0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979,
-  0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d,
-  0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9,
-  0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea,
-  0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808,
-  0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e,
-  0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6,
-  0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f,
-  0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a,
-  0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666,
-  0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e,
-  0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9,
-  0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e,
-  0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111,
-  0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494,
-  0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9,
-  0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf,
-  0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d,
-  0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868,
-  0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f,
-  0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616,
-};
-
-__constant u32 te2[256] =
-{
-  0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b,
-  0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5,
-  0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b,
-  0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76,
-  0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d,
-  0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0,
-  0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af,
-  0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0,
-  0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26,
-  0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc,
-  0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1,
-  0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15,
-  0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3,
-  0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a,
-  0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2,
-  0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75,
-  0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a,
-  0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0,
-  0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3,
-  0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384,
-  0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed,
-  0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b,
-  0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239,
-  0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf,
-  0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb,
-  0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185,
-  0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f,
-  0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8,
-  0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f,
-  0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5,
-  0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221,
-  0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2,
-  0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec,
-  0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17,
-  0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d,
-  0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673,
-  0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc,
-  0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88,
-  0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814,
-  0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb,
-  0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a,
-  0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c,
-  0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462,
-  0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279,
-  0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d,
-  0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9,
-  0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea,
-  0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008,
-  0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e,
-  0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6,
-  0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f,
-  0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a,
-  0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66,
-  0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e,
-  0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9,
-  0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e,
-  0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211,
-  0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394,
-  0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9,
-  0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df,
-  0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d,
-  0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068,
-  0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f,
-  0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16,
-};
-
-__constant u32 te3[256] =
-{
-  0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6,
-  0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491,
-  0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56,
-  0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec,
-  0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa,
-  0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb,
-  0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45,
-  0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b,
-  0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c,
-  0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83,
-  0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9,
-  0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a,
-  0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d,
-  0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f,
-  0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf,
-  0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea,
-  0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34,
-  0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b,
-  0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d,
-  0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713,
-  0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1,
-  0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6,
-  0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72,
-  0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85,
-  0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed,
-  0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411,
-  0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe,
-  0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b,
-  0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05,
-  0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1,
-  0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342,
-  0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf,
-  0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3,
-  0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e,
-  0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a,
-  0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6,
-  0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3,
-  0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b,
-  0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28,
-  0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad,
-  0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14,
-  0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8,
-  0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4,
-  0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2,
-  0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da,
-  0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049,
-  0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf,
-  0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810,
-  0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c,
-  0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197,
-  0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e,
-  0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f,
-  0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc,
-  0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c,
-  0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069,
-  0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927,
-  0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322,
-  0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733,
-  0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9,
-  0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5,
-  0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a,
-  0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0,
-  0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e,
-  0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c,
-};
-
-__constant u32 te4[256] =
-{
-  0x63636363, 0x7c7c7c7c, 0x77777777, 0x7b7b7b7b,
-  0xf2f2f2f2, 0x6b6b6b6b, 0x6f6f6f6f, 0xc5c5c5c5,
-  0x30303030, 0x01010101, 0x67676767, 0x2b2b2b2b,
-  0xfefefefe, 0xd7d7d7d7, 0xabababab, 0x76767676,
-  0xcacacaca, 0x82828282, 0xc9c9c9c9, 0x7d7d7d7d,
-  0xfafafafa, 0x59595959, 0x47474747, 0xf0f0f0f0,
-  0xadadadad, 0xd4d4d4d4, 0xa2a2a2a2, 0xafafafaf,
-  0x9c9c9c9c, 0xa4a4a4a4, 0x72727272, 0xc0c0c0c0,
-  0xb7b7b7b7, 0xfdfdfdfd, 0x93939393, 0x26262626,
-  0x36363636, 0x3f3f3f3f, 0xf7f7f7f7, 0xcccccccc,
-  0x34343434, 0xa5a5a5a5, 0xe5e5e5e5, 0xf1f1f1f1,
-  0x71717171, 0xd8d8d8d8, 0x31313131, 0x15151515,
-  0x04040404, 0xc7c7c7c7, 0x23232323, 0xc3c3c3c3,
-  0x18181818, 0x96969696, 0x05050505, 0x9a9a9a9a,
-  0x07070707, 0x12121212, 0x80808080, 0xe2e2e2e2,
-  0xebebebeb, 0x27272727, 0xb2b2b2b2, 0x75757575,
-  0x09090909, 0x83838383, 0x2c2c2c2c, 0x1a1a1a1a,
-  0x1b1b1b1b, 0x6e6e6e6e, 0x5a5a5a5a, 0xa0a0a0a0,
-  0x52525252, 0x3b3b3b3b, 0xd6d6d6d6, 0xb3b3b3b3,
-  0x29292929, 0xe3e3e3e3, 0x2f2f2f2f, 0x84848484,
-  0x53535353, 0xd1d1d1d1, 0x00000000, 0xedededed,
-  0x20202020, 0xfcfcfcfc, 0xb1b1b1b1, 0x5b5b5b5b,
-  0x6a6a6a6a, 0xcbcbcbcb, 0xbebebebe, 0x39393939,
-  0x4a4a4a4a, 0x4c4c4c4c, 0x58585858, 0xcfcfcfcf,
-  0xd0d0d0d0, 0xefefefef, 0xaaaaaaaa, 0xfbfbfbfb,
-  0x43434343, 0x4d4d4d4d, 0x33333333, 0x85858585,
-  0x45454545, 0xf9f9f9f9, 0x02020202, 0x7f7f7f7f,
-  0x50505050, 0x3c3c3c3c, 0x9f9f9f9f, 0xa8a8a8a8,
-  0x51515151, 0xa3a3a3a3, 0x40404040, 0x8f8f8f8f,
-  0x92929292, 0x9d9d9d9d, 0x38383838, 0xf5f5f5f5,
-  0xbcbcbcbc, 0xb6b6b6b6, 0xdadadada, 0x21212121,
-  0x10101010, 0xffffffff, 0xf3f3f3f3, 0xd2d2d2d2,
-  0xcdcdcdcd, 0x0c0c0c0c, 0x13131313, 0xecececec,
-  0x5f5f5f5f, 0x97979797, 0x44444444, 0x17171717,
-  0xc4c4c4c4, 0xa7a7a7a7, 0x7e7e7e7e, 0x3d3d3d3d,
-  0x64646464, 0x5d5d5d5d, 0x19191919, 0x73737373,
-  0x60606060, 0x81818181, 0x4f4f4f4f, 0xdcdcdcdc,
-  0x22222222, 0x2a2a2a2a, 0x90909090, 0x88888888,
-  0x46464646, 0xeeeeeeee, 0xb8b8b8b8, 0x14141414,
-  0xdededede, 0x5e5e5e5e, 0x0b0b0b0b, 0xdbdbdbdb,
-  0xe0e0e0e0, 0x32323232, 0x3a3a3a3a, 0x0a0a0a0a,
-  0x49494949, 0x06060606, 0x24242424, 0x5c5c5c5c,
-  0xc2c2c2c2, 0xd3d3d3d3, 0xacacacac, 0x62626262,
-  0x91919191, 0x95959595, 0xe4e4e4e4, 0x79797979,
-  0xe7e7e7e7, 0xc8c8c8c8, 0x37373737, 0x6d6d6d6d,
-  0x8d8d8d8d, 0xd5d5d5d5, 0x4e4e4e4e, 0xa9a9a9a9,
-  0x6c6c6c6c, 0x56565656, 0xf4f4f4f4, 0xeaeaeaea,
-  0x65656565, 0x7a7a7a7a, 0xaeaeaeae, 0x08080808,
-  0xbabababa, 0x78787878, 0x25252525, 0x2e2e2e2e,
-  0x1c1c1c1c, 0xa6a6a6a6, 0xb4b4b4b4, 0xc6c6c6c6,
-  0xe8e8e8e8, 0xdddddddd, 0x74747474, 0x1f1f1f1f,
-  0x4b4b4b4b, 0xbdbdbdbd, 0x8b8b8b8b, 0x8a8a8a8a,
-  0x70707070, 0x3e3e3e3e, 0xb5b5b5b5, 0x66666666,
-  0x48484848, 0x03030303, 0xf6f6f6f6, 0x0e0e0e0e,
-  0x61616161, 0x35353535, 0x57575757, 0xb9b9b9b9,
-  0x86868686, 0xc1c1c1c1, 0x1d1d1d1d, 0x9e9e9e9e,
-  0xe1e1e1e1, 0xf8f8f8f8, 0x98989898, 0x11111111,
-  0x69696969, 0xd9d9d9d9, 0x8e8e8e8e, 0x94949494,
-  0x9b9b9b9b, 0x1e1e1e1e, 0x87878787, 0xe9e9e9e9,
-  0xcececece, 0x55555555, 0x28282828, 0xdfdfdfdf,
-  0x8c8c8c8c, 0xa1a1a1a1, 0x89898989, 0x0d0d0d0d,
-  0xbfbfbfbf, 0xe6e6e6e6, 0x42424242, 0x68686868,
-  0x41414141, 0x99999999, 0x2d2d2d2d, 0x0f0f0f0f,
-  0xb0b0b0b0, 0x54545454, 0xbbbbbbbb, 0x16161616,
-};
-
-__constant u32 td0[256] =
-{
-  0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96,
-  0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393,
-  0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25,
-  0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f,
-  0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1,
-  0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6,
-  0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da,
-  0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844,
-  0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd,
-  0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4,
-  0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45,
-  0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94,
-  0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7,
-  0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a,
-  0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5,
-  0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c,
-  0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1,
-  0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a,
-  0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75,
-  0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051,
-  0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46,
-  0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff,
-  0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77,
-  0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb,
-  0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000,
-  0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e,
-  0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927,
-  0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a,
-  0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e,
-  0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16,
-  0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d,
-  0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8,
-  0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd,
-  0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34,
-  0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163,
-  0xd731dcca, 0x42638510, 0x13972240, 0x84c61120,
-  0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d,
-  0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0,
-  0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422,
-  0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef,
-  0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36,
-  0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4,
-  0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662,
-  0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5,
-  0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3,
-  0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b,
-  0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8,
-  0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6,
-  0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6,
-  0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0,
-  0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815,
-  0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f,
-  0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df,
-  0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f,
-  0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e,
-  0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713,
-  0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89,
-  0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c,
-  0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf,
-  0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86,
-  0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f,
-  0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541,
-  0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190,
-  0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742,
-};
-
-__constant u32 td1[256] =
-{
-  0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e,
-  0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303,
-  0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c,
-  0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3,
-  0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0,
-  0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9,
-  0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259,
-  0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8,
-  0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971,
-  0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a,
-  0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f,
-  0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b,
-  0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8,
-  0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab,
-  0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708,
-  0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682,
-  0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2,
-  0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe,
-  0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb,
-  0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10,
-  0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd,
-  0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015,
-  0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e,
-  0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee,
-  0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000,
-  0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72,
-  0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39,
-  0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e,
-  0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91,
-  0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a,
-  0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17,
-  0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9,
-  0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60,
-  0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e,
-  0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1,
-  0xcad731dc, 0x10426385, 0x40139722, 0x2084c611,
-  0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1,
-  0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3,
-  0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964,
-  0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390,
-  0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b,
-  0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf,
-  0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46,
-  0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af,
-  0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512,
-  0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb,
-  0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a,
-  0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8,
-  0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c,
-  0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266,
-  0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8,
-  0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6,
-  0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604,
-  0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551,
-  0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41,
-  0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647,
-  0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c,
-  0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1,
-  0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737,
-  0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db,
-  0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340,
-  0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95,
-  0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1,
-  0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857,
-};
-
-__constant u32 td2[256] =
-{
-  0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27,
-  0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3,
-  0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502,
-  0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562,
-  0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe,
-  0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3,
-  0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552,
-  0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9,
-  0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9,
-  0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce,
-  0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253,
-  0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908,
-  0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b,
-  0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655,
-  0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337,
-  0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16,
-  0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69,
-  0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6,
-  0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6,
-  0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e,
-  0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6,
-  0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050,
-  0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9,
-  0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8,
-  0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000,
-  0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a,
-  0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d,
-  0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436,
-  0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b,
-  0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12,
-  0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b,
-  0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e,
-  0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f,
-  0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb,
-  0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4,
-  0xdccad731, 0x85104263, 0x22401397, 0x112084c6,
-  0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729,
-  0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1,
-  0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9,
-  0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233,
-  0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4,
-  0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad,
-  0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e,
-  0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3,
-  0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25,
-  0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b,
-  0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f,
-  0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15,
-  0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0,
-  0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2,
-  0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7,
-  0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791,
-  0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496,
-  0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665,
-  0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b,
-  0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6,
-  0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13,
-  0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47,
-  0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7,
-  0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844,
-  0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3,
-  0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d,
-  0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456,
-  0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8,
-};
-
-__constant u32 td3[256] =
-{
-  0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a,
-  0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b,
-  0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5,
-  0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5,
-  0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d,
-  0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b,
-  0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95,
-  0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e,
-  0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27,
-  0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d,
-  0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562,
-  0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9,
-  0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752,
-  0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66,
-  0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3,
-  0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced,
-  0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e,
-  0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4,
-  0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4,
-  0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd,
-  0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d,
-  0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60,
-  0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767,
-  0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79,
-  0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000,
-  0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c,
-  0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736,
-  0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24,
-  0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b,
-  0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c,
-  0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12,
-  0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814,
-  0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3,
-  0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b,
-  0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8,
-  0x31dccad7, 0x63851042, 0x97224013, 0xc6112084,
-  0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7,
-  0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077,
-  0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247,
-  0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22,
-  0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698,
-  0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f,
-  0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254,
-  0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582,
-  0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf,
-  0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb,
-  0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883,
-  0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef,
-  0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629,
-  0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035,
-  0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533,
-  0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17,
-  0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4,
-  0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46,
-  0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb,
-  0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d,
-  0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb,
-  0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a,
-  0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73,
-  0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678,
-  0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2,
-  0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff,
-  0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064,
-  0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0,
-};
-
-__constant u32 td4[256] =
-{
-  0x52525252, 0x09090909, 0x6a6a6a6a, 0xd5d5d5d5,
-  0x30303030, 0x36363636, 0xa5a5a5a5, 0x38383838,
-  0xbfbfbfbf, 0x40404040, 0xa3a3a3a3, 0x9e9e9e9e,
-  0x81818181, 0xf3f3f3f3, 0xd7d7d7d7, 0xfbfbfbfb,
-  0x7c7c7c7c, 0xe3e3e3e3, 0x39393939, 0x82828282,
-  0x9b9b9b9b, 0x2f2f2f2f, 0xffffffff, 0x87878787,
-  0x34343434, 0x8e8e8e8e, 0x43434343, 0x44444444,
-  0xc4c4c4c4, 0xdededede, 0xe9e9e9e9, 0xcbcbcbcb,
-  0x54545454, 0x7b7b7b7b, 0x94949494, 0x32323232,
-  0xa6a6a6a6, 0xc2c2c2c2, 0x23232323, 0x3d3d3d3d,
-  0xeeeeeeee, 0x4c4c4c4c, 0x95959595, 0x0b0b0b0b,
-  0x42424242, 0xfafafafa, 0xc3c3c3c3, 0x4e4e4e4e,
-  0x08080808, 0x2e2e2e2e, 0xa1a1a1a1, 0x66666666,
-  0x28282828, 0xd9d9d9d9, 0x24242424, 0xb2b2b2b2,
-  0x76767676, 0x5b5b5b5b, 0xa2a2a2a2, 0x49494949,
-  0x6d6d6d6d, 0x8b8b8b8b, 0xd1d1d1d1, 0x25252525,
-  0x72727272, 0xf8f8f8f8, 0xf6f6f6f6, 0x64646464,
-  0x86868686, 0x68686868, 0x98989898, 0x16161616,
-  0xd4d4d4d4, 0xa4a4a4a4, 0x5c5c5c5c, 0xcccccccc,
-  0x5d5d5d5d, 0x65656565, 0xb6b6b6b6, 0x92929292,
-  0x6c6c6c6c, 0x70707070, 0x48484848, 0x50505050,
-  0xfdfdfdfd, 0xedededed, 0xb9b9b9b9, 0xdadadada,
-  0x5e5e5e5e, 0x15151515, 0x46464646, 0x57575757,
-  0xa7a7a7a7, 0x8d8d8d8d, 0x9d9d9d9d, 0x84848484,
-  0x90909090, 0xd8d8d8d8, 0xabababab, 0x00000000,
-  0x8c8c8c8c, 0xbcbcbcbc, 0xd3d3d3d3, 0x0a0a0a0a,
-  0xf7f7f7f7, 0xe4e4e4e4, 0x58585858, 0x05050505,
-  0xb8b8b8b8, 0xb3b3b3b3, 0x45454545, 0x06060606,
-  0xd0d0d0d0, 0x2c2c2c2c, 0x1e1e1e1e, 0x8f8f8f8f,
-  0xcacacaca, 0x3f3f3f3f, 0x0f0f0f0f, 0x02020202,
-  0xc1c1c1c1, 0xafafafaf, 0xbdbdbdbd, 0x03030303,
-  0x01010101, 0x13131313, 0x8a8a8a8a, 0x6b6b6b6b,
-  0x3a3a3a3a, 0x91919191, 0x11111111, 0x41414141,
-  0x4f4f4f4f, 0x67676767, 0xdcdcdcdc, 0xeaeaeaea,
-  0x97979797, 0xf2f2f2f2, 0xcfcfcfcf, 0xcececece,
-  0xf0f0f0f0, 0xb4b4b4b4, 0xe6e6e6e6, 0x73737373,
-  0x96969696, 0xacacacac, 0x74747474, 0x22222222,
-  0xe7e7e7e7, 0xadadadad, 0x35353535, 0x85858585,
-  0xe2e2e2e2, 0xf9f9f9f9, 0x37373737, 0xe8e8e8e8,
-  0x1c1c1c1c, 0x75757575, 0xdfdfdfdf, 0x6e6e6e6e,
-  0x47474747, 0xf1f1f1f1, 0x1a1a1a1a, 0x71717171,
-  0x1d1d1d1d, 0x29292929, 0xc5c5c5c5, 0x89898989,
-  0x6f6f6f6f, 0xb7b7b7b7, 0x62626262, 0x0e0e0e0e,
-  0xaaaaaaaa, 0x18181818, 0xbebebebe, 0x1b1b1b1b,
-  0xfcfcfcfc, 0x56565656, 0x3e3e3e3e, 0x4b4b4b4b,
-  0xc6c6c6c6, 0xd2d2d2d2, 0x79797979, 0x20202020,
-  0x9a9a9a9a, 0xdbdbdbdb, 0xc0c0c0c0, 0xfefefefe,
-  0x78787878, 0xcdcdcdcd, 0x5a5a5a5a, 0xf4f4f4f4,
-  0x1f1f1f1f, 0xdddddddd, 0xa8a8a8a8, 0x33333333,
-  0x88888888, 0x07070707, 0xc7c7c7c7, 0x31313131,
-  0xb1b1b1b1, 0x12121212, 0x10101010, 0x59595959,
-  0x27272727, 0x80808080, 0xecececec, 0x5f5f5f5f,
-  0x60606060, 0x51515151, 0x7f7f7f7f, 0xa9a9a9a9,
-  0x19191919, 0xb5b5b5b5, 0x4a4a4a4a, 0x0d0d0d0d,
-  0x2d2d2d2d, 0xe5e5e5e5, 0x7a7a7a7a, 0x9f9f9f9f,
-  0x93939393, 0xc9c9c9c9, 0x9c9c9c9c, 0xefefefef,
-  0xa0a0a0a0, 0xe0e0e0e0, 0x3b3b3b3b, 0x4d4d4d4d,
-  0xaeaeaeae, 0x2a2a2a2a, 0xf5f5f5f5, 0xb0b0b0b0,
-  0xc8c8c8c8, 0xebebebeb, 0xbbbbbbbb, 0x3c3c3c3c,
-  0x83838383, 0x53535353, 0x99999999, 0x61616161,
-  0x17171717, 0x2b2b2b2b, 0x04040404, 0x7e7e7e7e,
-  0xbabababa, 0x77777777, 0xd6d6d6d6, 0x26262626,
-  0xe1e1e1e1, 0x69696969, 0x14141414, 0x63636363,
-  0x55555555, 0x21212121, 0x0c0c0c0c, 0x7d7d7d7d,
-};
-
-__constant u32 rcon[] =
-{
-  0x01000000, 0x02000000, 0x04000000, 0x08000000,
-  0x10000000, 0x20000000, 0x40000000, 0x80000000,
-  0x1b000000, 0x36000000,
-};
-
-void aes256_ExpandKey (u32 *ks, const u32 *ukey)
-{
-  ks[0] = ukey[0];
-  ks[1] = ukey[1];
-  ks[2] = ukey[2];
-  ks[3] = ukey[3];
-  ks[4] = ukey[4];
-  ks[5] = ukey[5];
-  ks[6] = ukey[6];
-  ks[7] = ukey[7];
-
-  int i;
-  int j;
-
-  i = 0;
-  j = 0;
-
-  while (1)
-  {
-    u32 temp = ks[j +  7];
-
-    ks[j +  8] = ks[j +  0]
-           ^ (te2[(temp >> 16) & 0xff] & 0xff000000)
-           ^ (te3[(temp >>  8) & 0xff] & 0x00ff0000)
-           ^ (te0[(temp >>  0) & 0xff] & 0x0000ff00)
-           ^ (te1[(temp >> 24) & 0xff] & 0x000000ff)
-           ^ rcon[i];
-
-    ks[j +  9] = ks[j +  1] ^ ks[j +  8];
-    ks[j + 10] = ks[j +  2] ^ ks[j +  9];
-    ks[j + 11] = ks[j +  3] ^ ks[j + 10];
-
-    if (++i == 7) break;
-
-    temp = ks[j + 11];
-
-    ks[j + 12] = ks[j +  4]
-           ^ (te2[(temp >> 24) & 0xff] & 0xff000000)
-           ^ (te3[(temp >> 16) & 0xff] & 0x00ff0000)
-           ^ (te0[(temp >>  8) & 0xff] & 0x0000ff00)
-           ^ (te1[(temp >>  0) & 0xff] & 0x000000ff);
-
-    ks[j + 13] = ks[j +  5] ^ ks[j + 12];
-    ks[j + 14] = ks[j +  6] ^ ks[j + 13];
-    ks[j + 15] = ks[j +  7] ^ ks[j + 14];
-
-    j += 8;
-  }
-}
-
-void aes256_InvertKey (u32 *ks)
-{
-  for (u32 i = 0, j = 56; i < j; i += 4, j -= 4)
-  {
-    u32 temp;
-
-    temp = ks[i + 0]; ks[i + 0] = ks[j + 0]; ks[j + 0] = temp;
-    temp = ks[i + 1]; ks[i + 1] = ks[j + 1]; ks[j + 1] = temp;
-    temp = ks[i + 2]; ks[i + 2] = ks[j + 2]; ks[j + 2] = temp;
-    temp = ks[i + 3]; ks[i + 3] = ks[j + 3]; ks[j + 3] = temp;
-  }
-
-  for (u32 i = 1, j = 4; i < 14; i += 1, j += 4)
-  {
-    ks[j + 0] =
-      td0[te1[(ks[j + 0] >> 24) & 0xff] & 0xff] ^
-      td1[te1[(ks[j + 0] >> 16) & 0xff] & 0xff] ^
-      td2[te1[(ks[j + 0] >>  8) & 0xff] & 0xff] ^
-      td3[te1[(ks[j + 0] >>  0) & 0xff] & 0xff];
-
-    ks[j + 1] =
-      td0[te1[(ks[j + 1] >> 24) & 0xff] & 0xff] ^
-      td1[te1[(ks[j + 1] >> 16) & 0xff] & 0xff] ^
-      td2[te1[(ks[j + 1] >>  8) & 0xff] & 0xff] ^
-      td3[te1[(ks[j + 1] >>  0) & 0xff] & 0xff];
-
-    ks[j + 2] =
-      td0[te1[(ks[j + 2] >> 24) & 0xff] & 0xff] ^
-      td1[te1[(ks[j + 2] >> 16) & 0xff] & 0xff] ^
-      td2[te1[(ks[j + 2] >>  8) & 0xff] & 0xff] ^
-      td3[te1[(ks[j + 2] >>  0) & 0xff] & 0xff];
-
-    ks[j + 3] =
-      td0[te1[(ks[j + 3] >> 24) & 0xff] & 0xff] ^
-      td1[te1[(ks[j + 3] >> 16) & 0xff] & 0xff] ^
-      td2[te1[(ks[j + 3] >>  8) & 0xff] & 0xff] ^
-      td3[te1[(ks[j + 3] >>  0) & 0xff] & 0xff];
-  }
-}
-
-void aes256_set_encrypt_key (u32 *ks, const u32 *ukey)
-{
-  u32 ukey_s[8];
-
-  ukey_s[0] = swap32 (ukey[0]);
-  ukey_s[1] = swap32 (ukey[1]);
-  ukey_s[2] = swap32 (ukey[2]);
-  ukey_s[3] = swap32 (ukey[3]);
-  ukey_s[4] = swap32 (ukey[4]);
-  ukey_s[5] = swap32 (ukey[5]);
-  ukey_s[6] = swap32 (ukey[6]);
-  ukey_s[7] = swap32 (ukey[7]);
-
-  aes256_ExpandKey (ks, ukey_s);
-}
-
-void aes256_set_decrypt_key (u32 *ks, const u32 *ukey)
-{
-  u32 ukey_s[8];
-
-  ukey_s[0] = swap32 (ukey[0]);
-  ukey_s[1] = swap32 (ukey[1]);
-  ukey_s[2] = swap32 (ukey[2]);
-  ukey_s[3] = swap32 (ukey[3]);
-  ukey_s[4] = swap32 (ukey[4]);
-  ukey_s[5] = swap32 (ukey[5]);
-  ukey_s[6] = swap32 (ukey[6]);
-  ukey_s[7] = swap32 (ukey[7]);
-
-  aes256_ExpandKey (ks, ukey_s);
-
-  aes256_InvertKey (ks);
-}
-
-void aes256_decrypt (const u32 *ks, const u32 *in, u32 *out)
-{
-  u32 in_s[4];
-
-  in_s[0] = swap32 (in[0]);
-  in_s[1] = swap32 (in[1]);
-  in_s[2] = swap32 (in[2]);
-  in_s[3] = swap32 (in[3]);
-
-  u32 s0 = in_s[0] ^ ks[0];
-  u32 s1 = in_s[1] ^ ks[1];
-  u32 s2 = in_s[2] ^ ks[2];
-  u32 s3 = in_s[3] ^ ks[3];
-
-  u32 t0;
-  u32 t1;
-  u32 t2;
-  u32 t3;
-
-  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[ 4];
-  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[ 5];
-  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[ 6];
-  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[ 7];
-  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[ 8];
-  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[ 9];
-  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[10];
-  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[11];
-  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[12];
-  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[13];
-  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[14];
-  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[15];
-  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[16];
-  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[17];
-  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[18];
-  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[19];
-  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[20];
-  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[21];
-  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[22];
-  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[23];
-  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[24];
-  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[25];
-  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[26];
-  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[27];
-  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[28];
-  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[29];
-  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[30];
-  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[31];
-  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[32];
-  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[33];
-  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[34];
-  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[35];
-  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[36];
-  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[37];
-  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[38];
-  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[39];
-  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[40];
-  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[41];
-  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[42];
-  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[43];
-  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[44];
-  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[45];
-  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[46];
-  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[47];
-  s0 = td0[t0 >> 24] ^ td1[(t3 >> 16) & 0xff] ^ td2[(t2 >>  8) & 0xff] ^ td3[t1 & 0xff] ^ ks[48];
-  s1 = td0[t1 >> 24] ^ td1[(t0 >> 16) & 0xff] ^ td2[(t3 >>  8) & 0xff] ^ td3[t2 & 0xff] ^ ks[49];
-  s2 = td0[t2 >> 24] ^ td1[(t1 >> 16) & 0xff] ^ td2[(t0 >>  8) & 0xff] ^ td3[t3 & 0xff] ^ ks[50];
-  s3 = td0[t3 >> 24] ^ td1[(t2 >> 16) & 0xff] ^ td2[(t1 >>  8) & 0xff] ^ td3[t0 & 0xff] ^ ks[51];
-  t0 = td0[s0 >> 24] ^ td1[(s3 >> 16) & 0xff] ^ td2[(s2 >>  8) & 0xff] ^ td3[s1 & 0xff] ^ ks[52];
-  t1 = td0[s1 >> 24] ^ td1[(s0 >> 16) & 0xff] ^ td2[(s3 >>  8) & 0xff] ^ td3[s2 & 0xff] ^ ks[53];
-  t2 = td0[s2 >> 24] ^ td1[(s1 >> 16) & 0xff] ^ td2[(s0 >>  8) & 0xff] ^ td3[s3 & 0xff] ^ ks[54];
-  t3 = td0[s3 >> 24] ^ td1[(s2 >> 16) & 0xff] ^ td2[(s1 >>  8) & 0xff] ^ td3[s0 & 0xff] ^ ks[55];
-
-  out[0] = (td4[(t0 >> 24) & 0xff] & 0xff000000)
-         ^ (td4[(t3 >> 16) & 0xff] & 0x00ff0000)
-         ^ (td4[(t2 >>  8) & 0xff] & 0x0000ff00)
-         ^ (td4[(t1 >>  0) & 0xff] & 0x000000ff)
-         ^ ks[56];
-
-  out[1] = (td4[(t1 >> 24) & 0xff] & 0xff000000)
-         ^ (td4[(t0 >> 16) & 0xff] & 0x00ff0000)
-         ^ (td4[(t3 >>  8) & 0xff] & 0x0000ff00)
-         ^ (td4[(t2 >>  0) & 0xff] & 0x000000ff)
-         ^ ks[57];
-
-  out[2] = (td4[(t2 >> 24) & 0xff] & 0xff000000)
-         ^ (td4[(t1 >> 16) & 0xff] & 0x00ff0000)
-         ^ (td4[(t0 >>  8) & 0xff] & 0x0000ff00)
-         ^ (td4[(t3 >>  0) & 0xff] & 0x000000ff)
-         ^ ks[58];
-
-  out[3] = (td4[(t3 >> 24) & 0xff] & 0xff000000)
-         ^ (td4[(t2 >> 16) & 0xff] & 0x00ff0000)
-         ^ (td4[(t1 >>  8) & 0xff] & 0x0000ff00)
-         ^ (td4[(t0 >>  0) & 0xff] & 0x000000ff)
-         ^ ks[59];
-
-  out[0] = swap32 (out[0]);
-  out[1] = swap32 (out[1]);
-  out[2] = swap32 (out[2]);
-  out[3] = swap32 (out[3]);
-}
-
-void aes256_encrypt (const u32 *ks, const u32 *in, u32 *out)
-{
-  u32 in_s[4];
-
-  in_s[0] = swap32 (in[0]);
-  in_s[1] = swap32 (in[1]);
-  in_s[2] = swap32 (in[2]);
-  in_s[3] = swap32 (in[3]);
-
-  u32 s0 = in_s[0] ^ ks[0];
-  u32 s1 = in_s[1] ^ ks[1];
-  u32 s2 = in_s[2] ^ ks[2];
-  u32 s3 = in_s[3] ^ ks[3];
-
-  u32 t0;
-  u32 t1;
-  u32 t2;
-  u32 t3;
-
-  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[ 4];
-  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[ 5];
-  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[ 6];
-  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[ 7];
-  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[ 8];
-  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[ 9];
-  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[10];
-  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[11];
-  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[12];
-  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[13];
-  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[14];
-  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[15];
-  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[16];
-  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[17];
-  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[18];
-  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[19];
-  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[20];
-  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[21];
-  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[22];
-  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[23];
-  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[24];
-  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[25];
-  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[26];
-  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[27];
-  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[28];
-  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[29];
-  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[30];
-  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[31];
-  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[32];
-  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[33];
-  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[34];
-  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[35];
-  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[36];
-  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[37];
-  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[38];
-  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[39];
-  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[40];
-  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[41];
-  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[42];
-  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[43];
-  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[44];
-  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[45];
-  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[46];
-  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[47];
-  s0 = te0[t0 >> 24] ^ te1[(t1 >> 16) & 0xff] ^ te2[(t2 >>  8) & 0xff] ^ te3[t3 & 0xff] ^ ks[48];
-  s1 = te0[t1 >> 24] ^ te1[(t2 >> 16) & 0xff] ^ te2[(t3 >>  8) & 0xff] ^ te3[t0 & 0xff] ^ ks[49];
-  s2 = te0[t2 >> 24] ^ te1[(t3 >> 16) & 0xff] ^ te2[(t0 >>  8) & 0xff] ^ te3[t1 & 0xff] ^ ks[50];
-  s3 = te0[t3 >> 24] ^ te1[(t0 >> 16) & 0xff] ^ te2[(t1 >>  8) & 0xff] ^ te3[t2 & 0xff] ^ ks[51];
-  t0 = te0[s0 >> 24] ^ te1[(s1 >> 16) & 0xff] ^ te2[(s2 >>  8) & 0xff] ^ te3[s3 & 0xff] ^ ks[52];
-  t1 = te0[s1 >> 24] ^ te1[(s2 >> 16) & 0xff] ^ te2[(s3 >>  8) & 0xff] ^ te3[s0 & 0xff] ^ ks[53];
-  t2 = te0[s2 >> 24] ^ te1[(s3 >> 16) & 0xff] ^ te2[(s0 >>  8) & 0xff] ^ te3[s1 & 0xff] ^ ks[54];
-  t3 = te0[s3 >> 24] ^ te1[(s0 >> 16) & 0xff] ^ te2[(s1 >>  8) & 0xff] ^ te3[s2 & 0xff] ^ ks[55];
-
-  out[0] = (te4[(t0 >> 24) & 0xff] & 0xff000000)
-         ^ (te4[(t1 >> 16) & 0xff] & 0x00ff0000)
-         ^ (te4[(t2 >>  8) & 0xff] & 0x0000ff00)
-         ^ (te4[(t3 >>  0) & 0xff] & 0x000000ff)
-         ^ ks[56];
-
-  out[1] = (te4[(t1 >> 24) & 0xff] & 0xff000000)
-         ^ (te4[(t2 >> 16) & 0xff] & 0x00ff0000)
-         ^ (te4[(t3 >>  8) & 0xff] & 0x0000ff00)
-         ^ (te4[(t0 >>  0) & 0xff] & 0x000000ff)
-         ^ ks[57];
-
-  out[2] = (te4[(t2 >> 24) & 0xff] & 0xff000000)
-         ^ (te4[(t3 >> 16) & 0xff] & 0x00ff0000)
-         ^ (te4[(t0 >>  8) & 0xff] & 0x0000ff00)
-         ^ (te4[(t1 >>  0) & 0xff] & 0x000000ff)
-         ^ ks[58];
-
-  out[3] = (te4[(t3 >> 24) & 0xff] & 0xff000000)
-         ^ (te4[(t0 >> 16) & 0xff] & 0x00ff0000)
-         ^ (te4[(t1 >>  8) & 0xff] & 0x0000ff00)
-         ^ (te4[(t2 >>  0) & 0xff] & 0x000000ff)
-         ^ ks[59];
-
-  out[0] = swap32 (out[0]);
-  out[1] = swap32 (out[1]);
-  out[2] = swap32 (out[2]);
-  out[3] = swap32 (out[3]);
-}
-
-void aes256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out)
-{
-  u32 T[4] = { 0 };
-  u32 Z[4] = { 0 };
-
-  out[0] = in[0];
-  out[1] = in[1];
-  out[2] = in[2];
-  out[3] = in[3];
-
-  u32 ks[60];
-
-  aes256_set_encrypt_key (ks, ukey2);
-  aes256_encrypt (ks, Z, T);
-
-  out[0] ^= T[0];
-  out[1] ^= T[1];
-  out[2] ^= T[2];
-  out[3] ^= T[3];
-
-  aes256_set_decrypt_key (ks, ukey1);
-  aes256_decrypt (ks, out, out);
-
-  out[0] ^= T[0];
-  out[1] ^= T[1];
-  out[2] ^= T[2];
-  out[3] ^= T[3];
-}

diff --git a/OpenCL/kernel_serpent256.c b/OpenCL/kernel_serpent256.c
deleted file mode 100644 (file)
index 462c0b9..0000000
--- a/OpenCL/kernel_serpent256.c
+++ /dev/null
@@ -1,587 +0,0 @@
-/* This is an independent implementation of the encryption algorithm:   */
-/*                                                                      */
-/*         Serpent by Ross Anderson, Eli Biham and Lars Knudsen         */
-/*                                                                      */
-/* which is a candidate algorithm in the Advanced Encryption Standard   */
-/* programme of the US National Institute of Standards and Technology.  */
-/*                                                                      */
-/* Copyright in this implementation is held by Dr B R Gladman but I     */
-/* hereby give permission for its free direct or derivative use subject */
-/* to acknowledgment of its origin and compliance with any conditions   */
-/* that the originators of the algorithm place on its exploitation.     */
-/*                                                                      */
-/* Dr Brian Gladman (gladman@seven77.demon.co.uk) 14th January 1999     */
-/*                                                                      */
-/* -------------------------------------------------------------------- */
-/*                                                                      */
-/* Cleaned and optimized for GPU use with hashcat by Jens Steube        */
-
-/* 15 terms */
-
-#define sb0(a,b,c,d,e,f,g,h)    \
-    t1 = a ^ d;     \
-    t2 = a & d;     \
-    t3 = c ^ t1;    \
-    t6 = b & t1;    \
-    t4 = b ^ t3;    \
-    t10 = ~t3;      \
-    h = t2 ^ t4;    \
-    t7 = a ^ t6;    \
-    t14 = ~t7;      \
-    t8 = c | t7;    \
-    t11 = t3 ^ t7;  \
-    g = t4 ^ t8;    \
-    t12 = h & t11;  \
-    f = t10 ^ t12;  \
-    e = t12 ^ t14
-
-/* 15 terms */
-
-#define ib0(a,b,c,d,e,f,g,h)    \
-    t1 = ~a;        \
-    t2 = a ^ b;     \
-    t3 = t1 | t2;   \
-    t4 = d ^ t3;    \
-    t7 = d & t2;    \
-    t5 = c ^ t4;    \
-    t8 = t1 ^ t7;   \
-    g = t2 ^ t5;    \
-    t11 = a & t4;   \
-    t9 = g & t8;    \
-    t14 = t5 ^ t8;  \
-    f = t4 ^ t9;    \
-    t12 = t5 | f;   \
-    h = t11 ^ t12;  \
-    e = h ^ t14
-
-/* 14 terms!  */
-
-#define sb1(a,b,c,d,e,f,g,h)    \
-    t1 = ~a;        \
-    t2 = b ^ t1;    \
-    t3 = a | t2;    \
-    t4 = d | t2;    \
-    t5 = c ^ t3;    \
-    g = d ^ t5;     \
-    t7 = b ^ t4;    \
-    t8 = t2 ^ g;    \
-    t9 = t5 & t7;   \
-    h = t8 ^ t9;    \
-    t11 = t5 ^ t7;  \
-    f = h ^ t11;    \
-    t13 = t8 & t11; \
-    e = t5 ^ t13
-
-/* 17 terms */
-
-#define ib1(a,b,c,d,e,f,g,h)    \
-    t1 = a ^ d;     \
-    t2 = a & b;     \
-    t3 = b ^ c;     \
-    t4 = a ^ t3;    \
-    t5 = b | d;     \
-    t7 = c | t1;    \
-    h = t4 ^ t5;    \
-    t8 = b ^ t7;    \
-    t11 = ~t2;      \
-    t9 = t4 & t8;   \
-    f = t1 ^ t9;    \
-    t13 = t9 ^ t11; \
-    t12 = h & f;    \
-    g = t12 ^ t13;  \
-    t15 = a & d;    \
-    t16 = c ^ t13;  \
-    e = t15 ^ t16
-
-/* 16 terms */
-
-#define sb2(a,b,c,d,e,f,g,h)    \
-    t1 = ~a;        \
-    t2 = b ^ d;     \
-    t3 = c & t1;    \
-    t13 = d | t1;   \
-    e = t2 ^ t3;    \
-    t5 = c ^ t1;    \
-    t6 = c ^ e;     \
-    t7 = b & t6;    \
-    t10 = e | t5;   \
-    h = t5 ^ t7;    \
-    t9 = d | t7;    \
-    t11 = t9 & t10; \
-    t14 = t2 ^ h;   \
-    g = a ^ t11;    \
-    t15 = g ^ t13;  \
-    f = t14 ^ t15
-
-/* 16 terms */
-
-#define ib2(a,b,c,d,e,f,g,h)    \
-    t1 = b ^ d;     \
-    t2 = ~t1;       \
-    t3 = a ^ c;     \
-    t4 = c ^ t1;    \
-    t7 = a | t2;    \
-    t5 = b & t4;    \
-    t8 = d ^ t7;    \
-    t11 = ~t4;      \
-    e = t3 ^ t5;    \
-    t9 = t3 | t8;   \
-    t14 = d & t11;  \
-    h = t1 ^ t9;    \
-    t12 = e | h;    \
-    f = t11 ^ t12;  \
-    t15 = t3 ^ t12; \
-    g = t14 ^ t15
-
-/* 17 terms */
-
-#define sb3(a,b,c,d,e,f,g,h)    \
-    t1 = a ^ c;     \
-    t2 = d ^ t1;    \
-    t3 = a & t2;    \
-    t4 = d ^ t3;    \
-    t5 = b & t4;    \
-    g = t2 ^ t5;    \
-    t7 = a | g;     \
-    t8 = b | d;     \
-    t11 = a | d;    \
-    t9 = t4 & t7;   \
-    f = t8 ^ t9;    \
-    t12 = b ^ t11;  \
-    t13 = g ^ t9;   \
-    t15 = t3 ^ t8;  \
-    h = t12 ^ t13;  \
-    t16 = c & t15;  \
-    e = t12 ^ t16
-
-/* 16 term solution that performs less well than 17 term one
-   in my environment (PPro/PII)
-
-#define sb3(a,b,c,d,e,f,g,h)    \
-    t1 = a ^ b;     \
-    t2 = a & c;     \
-    t3 = a | d;     \
-    t4 = c ^ d;     \
-    t5 = t1 & t3;   \
-    t6 = t2 | t5;   \
-    g = t4 ^ t6;    \
-    t8 = b ^ t3;    \
-    t9 = t6 ^ t8;   \
-    t10 = t4 & t9;  \
-    e = t1 ^ t10;   \
-    t12 = g & e;    \
-    f = t9 ^ t12;   \
-    t14 = b | d;    \
-    t15 = t4 ^ t12; \
-    h = t14 ^ t15
-*/
-
-/* 17 terms */
-
-#define ib3(a,b,c,d,e,f,g,h)    \
-    t1 = b ^ c;     \
-    t2 = b | c;     \
-    t3 = a ^ c;     \
-    t7 = a ^ d;     \
-    t4 = t2 ^ t3;   \
-    t5 = d | t4;    \
-    t9 = t2 ^ t7;   \
-    e = t1 ^ t5;    \
-    t8 = t1 | t5;   \
-    t11 = a & t4;   \
-    g = t8 ^ t9;    \
-    t12 = e | t9;   \
-    f = t11 ^ t12;  \
-    t14 = a & g;    \
-    t15 = t2 ^ t14; \
-    t16 = e & t15;  \
-    h = t4 ^ t16
-
-/* 15 terms */
-
-#define sb4(a,b,c,d,e,f,g,h)    \
-    t1 = a ^ d;     \
-    t2 = d & t1;    \
-    t3 = c ^ t2;    \
-    t4 = b | t3;    \
-    h = t1 ^ t4;    \
-    t6 = ~b;        \
-    t7 = t1 | t6;   \
-    e = t3 ^ t7;    \
-    t9 = a & e;     \
-    t10 = t1 ^ t6;  \
-    t11 = t4 & t10; \
-    g = t9 ^ t11;   \
-    t13 = a ^ t3;   \
-    t14 = t10 & g;  \
-    f = t13 ^ t14
-
-/* 17 terms */
-
-#define ib4(a,b,c,d,e,f,g,h)    \
-    t1 = c ^ d;     \
-    t2 = c | d;     \
-    t3 = b ^ t2;    \
-    t4 = a & t3;    \
-    f = t1 ^ t4;    \
-    t6 = a ^ d;     \
-    t7 = b | d;     \
-    t8 = t6 & t7;   \
-    h = t3 ^ t8;    \
-    t10 = ~a;       \
-    t11 = c ^ h;    \
-    t12 = t10 | t11;\
-    e = t3 ^ t12;   \
-    t14 = c | t4;   \
-    t15 = t7 ^ t14; \
-    t16 = h | t10;  \
-    g = t15 ^ t16
-
-/* 16 terms */
-
-#define sb5(a,b,c,d,e,f,g,h)    \
-    t1 = ~a;        \
-    t2 = a ^ b;     \
-    t3 = a ^ d;     \
-    t4 = c ^ t1;    \
-    t5 = t2 | t3;   \
-    e = t4 ^ t5;    \
-    t7 = d & e;     \
-    t8 = t2 ^ e;    \
-    t10 = t1 | e;   \
-    f = t7 ^ t8;    \
-    t11 = t2 | t7;  \
-    t12 = t3 ^ t10; \
-    t14 = b ^ t7;   \
-    g = t11 ^ t12;  \
-    t15 = f & t12;  \
-    h = t14 ^ t15
-
-/* 16 terms */
-
-#define ib5(a,b,c,d,e,f,g,h)    \
-    t1 = ~c;        \
-    t2 = b & t1;    \
-    t3 = d ^ t2;    \
-    t4 = a & t3;    \
-    t5 = b ^ t1;    \
-    h = t4 ^ t5;    \
-    t7 = b | h;     \
-    t8 = a & t7;    \
-    f = t3 ^ t8;    \
-    t10 = a | d;    \
-    t11 = t1 ^ t7;  \
-    e = t10 ^ t11;  \
-    t13 = a ^ c;    \
-    t14 = b & t10;  \
-    t15 = t4 | t13; \
-    g = t14 ^ t15
-
-/* 15 terms */
-
-#define sb6(a,b,c,d,e,f,g,h)    \
-    t1 = ~a;        \
-    t2 = a ^ d;     \
-    t3 = b ^ t2;    \
-    t4 = t1 | t2;   \
-    t5 = c ^ t4;    \
-    f = b ^ t5;     \
-    t13 = ~t5;      \
-    t7 = t2 | f;    \
-    t8 = d ^ t7;    \
-    t9 = t5 & t8;   \
-    g = t3 ^ t9;    \
-    t11 = t5 ^ t8;  \
-    e = g ^ t11;    \
-    t14 = t3 & t11; \
-    h = t13 ^ t14
-
-/* 15 terms */
-
-#define ib6(a,b,c,d,e,f,g,h)    \
-    t1 = ~a;        \
-    t2 = a ^ b;     \
-    t3 = c ^ t2;    \
-    t4 = c | t1;    \
-    t5 = d ^ t4;    \
-    t13 = d & t1;   \
-    f = t3 ^ t5;    \
-    t7 = t3 & t5;   \
-    t8 = t2 ^ t7;   \
-    t9 = b | t8;    \
-    h = t5 ^ t9;    \
-    t11 = b | h;    \
-    e = t8 ^ t11;   \
-    t14 = t3 ^ t11; \
-    g = t13 ^ t14
-
-/* 17 terms */
-
-#define sb7(a,b,c,d,e,f,g,h)    \
-    t1 = ~c;        \
-    t2 = b ^ c;     \
-    t3 = b | t1;    \
-    t4 = d ^ t3;    \
-    t5 = a & t4;    \
-    t7 = a ^ d;     \
-    h = t2 ^ t5;    \
-    t8 = b ^ t5;    \
-    t9 = t2 | t8;   \
-    t11 = d & t3;   \
-    f = t7 ^ t9;    \
-    t12 = t5 ^ f;   \
-    t15 = t1 | t4;  \
-    t13 = h & t12;  \
-    g = t11 ^ t13;  \
-    t16 = t12 ^ g;  \
-    e = t15 ^ t16
-
-/* 17 terms */
-
-#define ib7(a,b,c,d,e,f,g,h)    \
-    t1 = a & b;     \
-    t2 = a | b;     \
-    t3 = c | t1;    \
-    t4 = d & t2;    \
-    h = t3 ^ t4;    \
-    t6 = ~d;        \
-    t7 = b ^ t4;    \
-    t8 = h ^ t6;    \
-    t11 = c ^ t7;   \
-    t9 = t7 | t8;   \
-    f = a ^ t9;     \
-    t12 = d | f;    \
-    e = t11 ^ t12;  \
-    t14 = a & h;    \
-    t15 = t3 ^ f;   \
-    t16 = e ^ t14;  \
-    g = t15 ^ t16
-
-#define k_xor(r,a,b,c,d)    \
-    a ^= ks[4 * r +  8]; \
-    b ^= ks[4 * r +  9]; \
-    c ^= ks[4 * r + 10]; \
-    d ^= ks[4 * r + 11]
-
-#define k_set(r,a,b,c,d)    \
-    a = ks[4 * r +  8];  \
-    b = ks[4 * r +  9];  \
-    c = ks[4 * r + 10];  \
-    d = ks[4 * r + 11]
-
-#define k_get(r,a,b,c,d)    \
-    ks[4 * r +  8] = a;  \
-    ks[4 * r +  9] = b;  \
-    ks[4 * r + 10] = c;  \
-    ks[4 * r + 11] = d
-
-/* the linear transformation and its inverse    */
-
-#define rot(a,b,c,d)    \
-    a = rotl32(a, 13);    \
-    c = rotl32(c, 3);     \
-    d ^= c ^ (a << 3);  \
-    b ^= a ^ c;         \
-    d = rotl32(d, 7);     \
-    b = rotl32(b, 1);     \
-    a ^= b ^ d;         \
-    c ^= d ^ (b << 7);  \
-    a = rotl32(a, 5);     \
-    c = rotl32(c, 22)
-
-#define irot(a,b,c,d)   \
-    c = rotr32(c, 22);    \
-    a = rotr32(a, 5);     \
-    c ^= d ^ (b << 7);  \
-    a ^= b ^ d;         \
-    d = rotr32(d, 7);     \
-    b = rotr32(b, 1);     \
-    d ^= c ^ (a << 3);  \
-    b ^= a ^ c;         \
-    c = rotr32(c, 3);     \
-    a = rotr32(a, 13)
-
-void serpent256_set_key (u32 *ks, const u32 *ukey)
-{
-  #ifdef _unroll
-  #pragma unroll
-  #endif
-  for (int i = 0; i < 8; i++)
-  {
-    ks[i] = ukey[i];
-  }
-
-  #ifdef _unroll
-  #pragma unroll
-  #endif
-  for (int i = 0; i < 132; i++)
-  {
-    ks[i + 8] = rotl32 (ks[i + 7] ^ ks[i + 5] ^ ks[i + 3] ^ ks[i + 0] ^ 0x9e3779b9 ^ i, 11);
-  }
-
-  u32  a,b,c,d,e,f,g,h;
-  u32  t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16;
-
-  k_set( 0,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get( 0,e,f,g,h);
-  k_set( 1,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get( 1,e,f,g,h);
-  k_set( 2,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get( 2,e,f,g,h);
-  k_set( 3,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get( 3,e,f,g,h);
-  k_set( 4,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get( 4,e,f,g,h);
-  k_set( 5,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get( 5,e,f,g,h);
-  k_set( 6,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get( 6,e,f,g,h);
-  k_set( 7,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get( 7,e,f,g,h);
-  k_set( 8,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get( 8,e,f,g,h);
-  k_set( 9,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get( 9,e,f,g,h);
-  k_set(10,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(10,e,f,g,h);
-  k_set(11,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(11,e,f,g,h);
-  k_set(12,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(12,e,f,g,h);
-  k_set(13,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(13,e,f,g,h);
-  k_set(14,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(14,e,f,g,h);
-  k_set(15,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(15,e,f,g,h);
-  k_set(16,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(16,e,f,g,h);
-  k_set(17,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get(17,e,f,g,h);
-  k_set(18,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(18,e,f,g,h);
-  k_set(19,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(19,e,f,g,h);
-  k_set(20,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(20,e,f,g,h);
-  k_set(21,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(21,e,f,g,h);
-  k_set(22,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(22,e,f,g,h);
-  k_set(23,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(23,e,f,g,h);
-  k_set(24,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(24,e,f,g,h);
-  k_set(25,a,b,c,d); sb2(a,b,c,d,e,f,g,h); k_get(25,e,f,g,h);
-  k_set(26,a,b,c,d); sb1(a,b,c,d,e,f,g,h); k_get(26,e,f,g,h);
-  k_set(27,a,b,c,d); sb0(a,b,c,d,e,f,g,h); k_get(27,e,f,g,h);
-  k_set(28,a,b,c,d); sb7(a,b,c,d,e,f,g,h); k_get(28,e,f,g,h);
-  k_set(29,a,b,c,d); sb6(a,b,c,d,e,f,g,h); k_get(29,e,f,g,h);
-  k_set(30,a,b,c,d); sb5(a,b,c,d,e,f,g,h); k_get(30,e,f,g,h);
-  k_set(31,a,b,c,d); sb4(a,b,c,d,e,f,g,h); k_get(31,e,f,g,h);
-  k_set(32,a,b,c,d); sb3(a,b,c,d,e,f,g,h); k_get(32,e,f,g,h);
-}
-
-void serpent256_encrypt (const u32 *ks, const u32 *in, u32 *out)
-{
-  u32  a,b,c,d,e,f,g,h;
-  u32  t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16;
-
-  a = in[0];
-  b = in[1];
-  c = in[2];
-  d = in[3];
-
-  k_xor( 0,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor( 1,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor( 2,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor( 3,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor( 4,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor( 5,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor( 6,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor( 7,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor( 8,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor( 9,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(10,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(11,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(12,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(13,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(14,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(15,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(16,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(17,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(18,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(19,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(20,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(21,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(22,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(23,e,f,g,h); sb7(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(24,a,b,c,d); sb0(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(25,e,f,g,h); sb1(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(26,a,b,c,d); sb2(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(27,e,f,g,h); sb3(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(28,a,b,c,d); sb4(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(29,e,f,g,h); sb5(e,f,g,h,a,b,c,d); rot(a,b,c,d);
-  k_xor(30,a,b,c,d); sb6(a,b,c,d,e,f,g,h); rot(e,f,g,h);
-  k_xor(31,e,f,g,h); sb7(e,f,g,h,a,b,c,d);
-  k_xor(32,a,b,c,d);
-
-  out[0] = a;
-  out[1] = b;
-  out[2] = c;
-  out[3] = d;
-}
-
-void serpent256_decrypt (const u32 *ks, const u32 *in, u32 *out)
-{
-  u32  a,b,c,d,e,f,g,h;
-  u32  t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,t16;
-
-  a = in[0];
-  b = in[1];
-  c = in[2];
-  d = in[3];
-
-                                       k_xor(32,a,b,c,d);
-                 ib7(a,b,c,d,e,f,g,h); k_xor(31,e,f,g,h);
-  irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(30,a,b,c,d);
-  irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(29,e,f,g,h);
-  irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(28,a,b,c,d);
-  irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(27,e,f,g,h);
-  irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(26,a,b,c,d);
-  irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor(25,e,f,g,h);
-  irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor(24,a,b,c,d);
-  irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor(23,e,f,g,h);
-  irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(22,a,b,c,d);
-  irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(21,e,f,g,h);
-  irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(20,a,b,c,d);
-  irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(19,e,f,g,h);
-  irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(18,a,b,c,d);
-  irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor(17,e,f,g,h);
-  irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor(16,a,b,c,d);
-  irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor(15,e,f,g,h);
-  irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor(14,a,b,c,d);
-  irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor(13,e,f,g,h);
-  irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor(12,a,b,c,d);
-  irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor(11,e,f,g,h);
-  irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor(10,a,b,c,d);
-  irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor( 9,e,f,g,h);
-  irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor( 8,a,b,c,d);
-  irot(a,b,c,d); ib7(a,b,c,d,e,f,g,h); k_xor( 7,e,f,g,h);
-  irot(e,f,g,h); ib6(e,f,g,h,a,b,c,d); k_xor( 6,a,b,c,d);
-  irot(a,b,c,d); ib5(a,b,c,d,e,f,g,h); k_xor( 5,e,f,g,h);
-  irot(e,f,g,h); ib4(e,f,g,h,a,b,c,d); k_xor( 4,a,b,c,d);
-  irot(a,b,c,d); ib3(a,b,c,d,e,f,g,h); k_xor( 3,e,f,g,h);
-  irot(e,f,g,h); ib2(e,f,g,h,a,b,c,d); k_xor( 2,a,b,c,d);
-  irot(a,b,c,d); ib1(a,b,c,d,e,f,g,h); k_xor( 1,e,f,g,h);
-  irot(e,f,g,h); ib0(e,f,g,h,a,b,c,d); k_xor( 0,a,b,c,d);
-
-  out[0] = a;
-  out[1] = b;
-  out[2] = c;
-  out[3] = d;
-}
-
-void serpent256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out)
-{
-  u32 T[4] = { 0 };
-  u32 Z[4] = { 0 };
-
-  out[0] = in[0];
-  out[1] = in[1];
-  out[2] = in[2];
-  out[3] = in[3];
-
-  u32 ks[140];
-
-  serpent256_set_key (ks, ukey2);
-  serpent256_encrypt (ks, Z, T);
-
-  out[0] ^= T[0];
-  out[1] ^= T[1];
-  out[2] ^= T[2];
-  out[3] ^= T[3];
-
-  serpent256_set_key (ks, ukey1);
-  serpent256_decrypt (ks, out, out);
-
-  out[0] ^= T[0];
-  out[1] ^= T[1];
-  out[2] ^= T[2];
-  out[3] ^= T[3];
-}

diff --git a/OpenCL/kernel_twofish256.c b/OpenCL/kernel_twofish256.c
deleted file mode 100644 (file)
index 6482a63..0000000
--- a/OpenCL/kernel_twofish256.c
+++ /dev/null
@@ -1,466 +0,0 @@
-/* This is an independent implementation of the encryption algorithm:   */
-/*                                                                      */
-/*         Twofish by Bruce Schneier and colleagues                     */
-/*                                                                      */
-/* which is a candidate algorithm in the Advanced Encryption Standard   */
-/* programme of the US National Institute of Standards and Technology.  */
-/*                                                                      */
-/* Copyright in this implementation is held by Dr B R Gladman but I     */
-/* hereby give permission for its free direct or derivative use subject */
-/* to acknowledgment of its origin and compliance with any conditions   */
-/* that the originators of t he algorithm place on its exploitation.    */
-/*                                                                      */
-/* My thanks to Doug Whiting and Niels Ferguson for comments that led   */
-/* to improvements in this implementation.                              */
-/*                                                                      */
-/* Dr Brian Gladman (gladman@seven77.demon.co.uk) 14th January 1999     */
-/*                                                                      */
-/* -------------------------------------------------------------------- */
-/*                                                                      */
-/* Cleaned and optimized for GPU use with hashcat by Jens Steube        */
-
-#define extract_byte(x,n) (((x) >> (8 * (n))) & 0xff)
-
-__constant u32 q_tab[2][256] =
-{
-  {
-    0xA9, 0x67, 0xB3, 0xE8, 0x04, 0xFD, 0xA3, 0x76, 0x9A, 0x92, 0x80, 0x78,
-    0xE4, 0xDD, 0xD1, 0x38, 0x0D, 0xC6, 0x35, 0x98, 0x18, 0xF7, 0xEC, 0x6C,
-    0x43, 0x75, 0x37, 0x26, 0xFA, 0x13, 0x94, 0x48, 0xF2, 0xD0, 0x8B, 0x30,
-    0x84, 0x54, 0xDF, 0x23, 0x19, 0x5B, 0x3D, 0x59, 0xF3, 0xAE, 0xA2, 0x82,
-    0x63, 0x01, 0x83, 0x2E, 0xD9, 0x51, 0x9B, 0x7C, 0xA6, 0xEB, 0xA5, 0xBE,
-    0x16, 0x0C, 0xE3, 0x61, 0xC0, 0x8C, 0x3A, 0xF5, 0x73, 0x2C, 0x25, 0x0B,
-    0xBB, 0x4E, 0x89, 0x6B, 0x53, 0x6A, 0xB4, 0xF1, 0xE1, 0xE6, 0xBD, 0x45,
-    0xE2, 0xF4, 0xB6, 0x66, 0xCC, 0x95, 0x03, 0x56, 0xD4, 0x1C, 0x1E, 0xD7,
-    0xFB, 0xC3, 0x8E, 0xB5, 0xE9, 0xCF, 0xBF, 0xBA, 0xEA, 0x77, 0x39, 0xAF,
-    0x33, 0xC9, 0x62, 0x71, 0x81, 0x79, 0x09, 0xAD, 0x24, 0xCD, 0xF9, 0xD8,
-    0xE5, 0xC5, 0xB9, 0x4D, 0x44, 0x08, 0x86, 0xE7, 0xA1, 0x1D, 0xAA, 0xED,
-    0x06, 0x70, 0xB2, 0xD2, 0x41, 0x7B, 0xA0, 0x11, 0x31, 0xC2, 0x27, 0x90,
-    0x20, 0xF6, 0x60, 0xFF, 0x96, 0x5C, 0xB1, 0xAB, 0x9E, 0x9C, 0x52, 0x1B,
-    0x5F, 0x93, 0x0A, 0xEF, 0x91, 0x85, 0x49, 0xEE, 0x2D, 0x4F, 0x8F, 0x3B,
-    0x47, 0x87, 0x6D, 0x46, 0xD6, 0x3E, 0x69, 0x64, 0x2A, 0xCE, 0xCB, 0x2F,
-    0xFC, 0x97, 0x05, 0x7A, 0xAC, 0x7F, 0xD5, 0x1A, 0x4B, 0x0E, 0xA7, 0x5A,
-    0x28, 0x14, 0x3F, 0x29, 0x88, 0x3C, 0x4C, 0x02, 0xB8, 0xDA, 0xB0, 0x17,
-    0x55, 0x1F, 0x8A, 0x7D, 0x57, 0xC7, 0x8D, 0x74, 0xB7, 0xC4, 0x9F, 0x72,
-    0x7E, 0x15, 0x22, 0x12, 0x58, 0x07, 0x99, 0x34, 0x6E, 0x50, 0xDE, 0x68,
-    0x65, 0xBC, 0xDB, 0xF8, 0xC8, 0xA8, 0x2B, 0x40, 0xDC, 0xFE, 0x32, 0xA4,
-    0xCA, 0x10, 0x21, 0xF0, 0xD3, 0x5D, 0x0F, 0x00, 0x6F, 0x9D, 0x36, 0x42,
-    0x4A, 0x5E, 0xC1, 0xE0
-  },
-  {
-    0x75, 0xF3, 0xC6, 0xF4, 0xDB, 0x7B, 0xFB, 0xC8, 0x4A, 0xD3, 0xE6, 0x6B,
-    0x45, 0x7D, 0xE8, 0x4B, 0xD6, 0x32, 0xD8, 0xFD, 0x37, 0x71, 0xF1, 0xE1,
-    0x30, 0x0F, 0xF8, 0x1B, 0x87, 0xFA, 0x06, 0x3F, 0x5E, 0xBA, 0xAE, 0x5B,
-    0x8A, 0x00, 0xBC, 0x9D, 0x6D, 0xC1, 0xB1, 0x0E, 0x80, 0x5D, 0xD2, 0xD5,
-    0xA0, 0x84, 0x07, 0x14, 0xB5, 0x90, 0x2C, 0xA3, 0xB2, 0x73, 0x4C, 0x54,
-    0x92, 0x74, 0x36, 0x51, 0x38, 0xB0, 0xBD, 0x5A, 0xFC, 0x60, 0x62, 0x96,
-    0x6C, 0x42, 0xF7, 0x10, 0x7C, 0x28, 0x27, 0x8C, 0x13, 0x95, 0x9C, 0xC7,
-    0x24, 0x46, 0x3B, 0x70, 0xCA, 0xE3, 0x85, 0xCB, 0x11, 0xD0, 0x93, 0xB8,
-    0xA6, 0x83, 0x20, 0xFF, 0x9F, 0x77, 0xC3, 0xCC, 0x03, 0x6F, 0x08, 0xBF,
-    0x40, 0xE7, 0x2B, 0xE2, 0x79, 0x0C, 0xAA, 0x82, 0x41, 0x3A, 0xEA, 0xB9,
-    0xE4, 0x9A, 0xA4, 0x97, 0x7E, 0xDA, 0x7A, 0x17, 0x66, 0x94, 0xA1, 0x1D,
-    0x3D, 0xF0, 0xDE, 0xB3, 0x0B, 0x72, 0xA7, 0x1C, 0xEF, 0xD1, 0x53, 0x3E,
-    0x8F, 0x33, 0x26, 0x5F, 0xEC, 0x76, 0x2A, 0x49, 0x81, 0x88, 0xEE, 0x21,
-    0xC4, 0x1A, 0xEB, 0xD9, 0xC5, 0x39, 0x99, 0xCD, 0xAD, 0x31, 0x8B, 0x01,
-    0x18, 0x23, 0xDD, 0x1F, 0x4E, 0x2D, 0xF9, 0x48, 0x4F, 0xF2, 0x65, 0x8E,
-    0x78, 0x5C, 0x58, 0x19, 0x8D, 0xE5, 0x98, 0x57, 0x67, 0x7F, 0x05, 0x64,
-    0xAF, 0x63, 0xB6, 0xFE, 0xF5, 0xB7, 0x3C, 0xA5, 0xCE, 0xE9, 0x68, 0x44,
-    0xE0, 0x4D, 0x43, 0x69, 0x29, 0x2E, 0xAC, 0x15, 0x59, 0xA8, 0x0A, 0x9E,
-    0x6E, 0x47, 0xDF, 0x34, 0x35, 0x6A, 0xCF, 0xDC, 0x22, 0xC9, 0xC0, 0x9B,
-    0x89, 0xD4, 0xED, 0xAB, 0x12, 0xA2, 0x0D, 0x52, 0xBB, 0x02, 0x2F, 0xA9,
-    0xD7, 0x61, 0x1E, 0xB4, 0x50, 0x04, 0xF6, 0xC2, 0x16, 0x25, 0x86, 0x56,
-    0x55, 0x09, 0xBE, 0x91
-  }
-};
-
-#define q(n,x) q_tab[n][x]
-
-__constant u32  m_tab[4][256] =
-{
-  { 0xBCBC3275, 0xECEC21F3, 0x202043C6, 0xB3B3C9F4, 0xDADA03DB, 0x02028B7B,
-    0xE2E22BFB, 0x9E9EFAC8, 0xC9C9EC4A, 0xD4D409D3, 0x18186BE6, 0x1E1E9F6B,
-    0x98980E45, 0xB2B2387D, 0xA6A6D2E8, 0x2626B74B, 0x3C3C57D6, 0x93938A32,
-    0x8282EED8, 0x525298FD, 0x7B7BD437, 0xBBBB3771, 0x5B5B97F1, 0x474783E1,
-    0x24243C30, 0x5151E20F, 0xBABAC6F8, 0x4A4AF31B, 0xBFBF4887, 0x0D0D70FA,
-    0xB0B0B306, 0x7575DE3F, 0xD2D2FD5E, 0x7D7D20BA, 0x666631AE, 0x3A3AA35B,
-    0x59591C8A, 0x00000000, 0xCDCD93BC, 0x1A1AE09D, 0xAEAE2C6D, 0x7F7FABC1,
-    0x2B2BC7B1, 0xBEBEB90E, 0xE0E0A080, 0x8A8A105D, 0x3B3B52D2, 0x6464BAD5,
-    0xD8D888A0, 0xE7E7A584, 0x5F5FE807, 0x1B1B1114, 0x2C2CC2B5, 0xFCFCB490,
-    0x3131272C, 0x808065A3, 0x73732AB2, 0x0C0C8173, 0x79795F4C, 0x6B6B4154,
-    0x4B4B0292, 0x53536974, 0x94948F36, 0x83831F51, 0x2A2A3638, 0xC4C49CB0,
-    0x2222C8BD, 0xD5D5F85A, 0xBDBDC3FC, 0x48487860, 0xFFFFCE62, 0x4C4C0796,
-    0x4141776C, 0xC7C7E642, 0xEBEB24F7, 0x1C1C1410, 0x5D5D637C, 0x36362228,
-    0x6767C027, 0xE9E9AF8C, 0x4444F913, 0x1414EA95, 0xF5F5BB9C, 0xCFCF18C7,
-    0x3F3F2D24, 0xC0C0E346, 0x7272DB3B, 0x54546C70, 0x29294CCA, 0xF0F035E3,
-    0x0808FE85, 0xC6C617CB, 0xF3F34F11, 0x8C8CE4D0, 0xA4A45993, 0xCACA96B8,
-    0x68683BA6, 0xB8B84D83, 0x38382820, 0xE5E52EFF, 0xADAD569F, 0x0B0B8477,
-    0xC8C81DC3, 0x9999FFCC, 0x5858ED03, 0x19199A6F, 0x0E0E0A08, 0x95957EBF,
-    0x70705040, 0xF7F730E7, 0x6E6ECF2B, 0x1F1F6EE2, 0xB5B53D79, 0x09090F0C,
-    0x616134AA, 0x57571682, 0x9F9F0B41, 0x9D9D803A, 0x111164EA, 0x2525CDB9,
-    0xAFAFDDE4, 0x4545089A, 0xDFDF8DA4, 0xA3A35C97, 0xEAEAD57E, 0x353558DA,
-    0xEDEDD07A, 0x4343FC17, 0xF8F8CB66, 0xFBFBB194, 0x3737D3A1, 0xFAFA401D,
-    0xC2C2683D, 0xB4B4CCF0, 0x32325DDE, 0x9C9C71B3, 0x5656E70B, 0xE3E3DA72,
-    0x878760A7, 0x15151B1C, 0xF9F93AEF, 0x6363BFD1, 0x3434A953, 0x9A9A853E,
-    0xB1B1428F, 0x7C7CD133, 0x88889B26, 0x3D3DA65F, 0xA1A1D7EC, 0xE4E4DF76,
-    0x8181942A, 0x91910149, 0x0F0FFB81, 0xEEEEAA88, 0x161661EE, 0xD7D77321,
-    0x9797F5C4, 0xA5A5A81A, 0xFEFE3FEB, 0x6D6DB5D9, 0x7878AEC5, 0xC5C56D39,
-    0x1D1DE599, 0x7676A4CD, 0x3E3EDCAD, 0xCBCB6731, 0xB6B6478B, 0xEFEF5B01,
-    0x12121E18, 0x6060C523, 0x6A6AB0DD, 0x4D4DF61F, 0xCECEE94E, 0xDEDE7C2D,
-    0x55559DF9, 0x7E7E5A48, 0x2121B24F, 0x03037AF2, 0xA0A02665, 0x5E5E198E,
-    0x5A5A6678, 0x65654B5C, 0x62624E58, 0xFDFD4519, 0x0606F48D, 0x404086E5,
-    0xF2F2BE98, 0x3333AC57, 0x17179067, 0x05058E7F, 0xE8E85E05, 0x4F4F7D64,
-    0x89896AAF, 0x10109563, 0x74742FB6, 0x0A0A75FE, 0x5C5C92F5, 0x9B9B74B7,
-    0x2D2D333C, 0x3030D6A5, 0x2E2E49CE, 0x494989E9, 0x46467268, 0x77775544,
-    0xA8A8D8E0, 0x9696044D, 0x2828BD43, 0xA9A92969, 0xD9D97929, 0x8686912E,
-    0xD1D187AC, 0xF4F44A15, 0x8D8D1559, 0xD6D682A8, 0xB9B9BC0A, 0x42420D9E,
-    0xF6F6C16E, 0x2F2FB847, 0xDDDD06DF, 0x23233934, 0xCCCC6235, 0xF1F1C46A,
-    0xC1C112CF, 0x8585EBDC, 0x8F8F9E22, 0x7171A1C9, 0x9090F0C0, 0xAAAA539B,
-    0x0101F189, 0x8B8BE1D4, 0x4E4E8CED, 0x8E8E6FAB, 0xABABA212, 0x6F6F3EA2,
-    0xE6E6540D, 0xDBDBF252, 0x92927BBB, 0xB7B7B602, 0x6969CA2F, 0x3939D9A9,
-    0xD3D30CD7, 0xA7A72361, 0xA2A2AD1E, 0xC3C399B4, 0x6C6C4450, 0x07070504,
-    0x04047FF6, 0x272746C2, 0xACACA716, 0xD0D07625, 0x50501386, 0xDCDCF756,
-    0x84841A55, 0xE1E15109, 0x7A7A25BE, 0x1313EF91 },
-
-  { 0xA9D93939, 0x67901717, 0xB3719C9C, 0xE8D2A6A6, 0x04050707, 0xFD985252,
-    0xA3658080, 0x76DFE4E4, 0x9A084545, 0x92024B4B, 0x80A0E0E0, 0x78665A5A,
-    0xE4DDAFAF, 0xDDB06A6A, 0xD1BF6363, 0x38362A2A, 0x0D54E6E6, 0xC6432020,
-    0x3562CCCC, 0x98BEF2F2, 0x181E1212, 0xF724EBEB, 0xECD7A1A1, 0x6C774141,
-    0x43BD2828, 0x7532BCBC, 0x37D47B7B, 0x269B8888, 0xFA700D0D, 0x13F94444,
-    0x94B1FBFB, 0x485A7E7E, 0xF27A0303, 0xD0E48C8C, 0x8B47B6B6, 0x303C2424,
-    0x84A5E7E7, 0x54416B6B, 0xDF06DDDD, 0x23C56060, 0x1945FDFD, 0x5BA33A3A,
-    0x3D68C2C2, 0x59158D8D, 0xF321ECEC, 0xAE316666, 0xA23E6F6F, 0x82165757,
-    0x63951010, 0x015BEFEF, 0x834DB8B8, 0x2E918686, 0xD9B56D6D, 0x511F8383,
-    0x9B53AAAA, 0x7C635D5D, 0xA63B6868, 0xEB3FFEFE, 0xA5D63030, 0xBE257A7A,
-    0x16A7ACAC, 0x0C0F0909, 0xE335F0F0, 0x6123A7A7, 0xC0F09090, 0x8CAFE9E9,
-    0x3A809D9D, 0xF5925C5C, 0x73810C0C, 0x2C273131, 0x2576D0D0, 0x0BE75656,
-    0xBB7B9292, 0x4EE9CECE, 0x89F10101, 0x6B9F1E1E, 0x53A93434, 0x6AC4F1F1,
-    0xB499C3C3, 0xF1975B5B, 0xE1834747, 0xE66B1818, 0xBDC82222, 0x450E9898,
-    0xE26E1F1F, 0xF4C9B3B3, 0xB62F7474, 0x66CBF8F8, 0xCCFF9999, 0x95EA1414,
-    0x03ED5858, 0x56F7DCDC, 0xD4E18B8B, 0x1C1B1515, 0x1EADA2A2, 0xD70CD3D3,
-    0xFB2BE2E2, 0xC31DC8C8, 0x8E195E5E, 0xB5C22C2C, 0xE9894949, 0xCF12C1C1,
-    0xBF7E9595, 0xBA207D7D, 0xEA641111, 0x77840B0B, 0x396DC5C5, 0xAF6A8989,
-    0x33D17C7C, 0xC9A17171, 0x62CEFFFF, 0x7137BBBB, 0x81FB0F0F, 0x793DB5B5,
-    0x0951E1E1, 0xADDC3E3E, 0x242D3F3F, 0xCDA47676, 0xF99D5555, 0xD8EE8282,
-    0xE5864040, 0xC5AE7878, 0xB9CD2525, 0x4D049696, 0x44557777, 0x080A0E0E,
-    0x86135050, 0xE730F7F7, 0xA1D33737, 0x1D40FAFA, 0xAA346161, 0xED8C4E4E,
-    0x06B3B0B0, 0x706C5454, 0xB22A7373, 0xD2523B3B, 0x410B9F9F, 0x7B8B0202,
-    0xA088D8D8, 0x114FF3F3, 0x3167CBCB, 0xC2462727, 0x27C06767, 0x90B4FCFC,
-    0x20283838, 0xF67F0404, 0x60784848, 0xFF2EE5E5, 0x96074C4C, 0x5C4B6565,
-    0xB1C72B2B, 0xAB6F8E8E, 0x9E0D4242, 0x9CBBF5F5, 0x52F2DBDB, 0x1BF34A4A,
-    0x5FA63D3D, 0x9359A4A4, 0x0ABCB9B9, 0xEF3AF9F9, 0x91EF1313, 0x85FE0808,
-    0x49019191, 0xEE611616, 0x2D7CDEDE, 0x4FB22121, 0x8F42B1B1, 0x3BDB7272,
-    0x47B82F2F, 0x8748BFBF, 0x6D2CAEAE, 0x46E3C0C0, 0xD6573C3C, 0x3E859A9A,
-    0x6929A9A9, 0x647D4F4F, 0x2A948181, 0xCE492E2E, 0xCB17C6C6, 0x2FCA6969,
-    0xFCC3BDBD, 0x975CA3A3, 0x055EE8E8, 0x7AD0EDED, 0xAC87D1D1, 0x7F8E0505,
-    0xD5BA6464, 0x1AA8A5A5, 0x4BB72626, 0x0EB9BEBE, 0xA7608787, 0x5AF8D5D5,
-    0x28223636, 0x14111B1B, 0x3FDE7575, 0x2979D9D9, 0x88AAEEEE, 0x3C332D2D,
-    0x4C5F7979, 0x02B6B7B7, 0xB896CACA, 0xDA583535, 0xB09CC4C4, 0x17FC4343,
-    0x551A8484, 0x1FF64D4D, 0x8A1C5959, 0x7D38B2B2, 0x57AC3333, 0xC718CFCF,
-    0x8DF40606, 0x74695353, 0xB7749B9B, 0xC4F59797, 0x9F56ADAD, 0x72DAE3E3,
-    0x7ED5EAEA, 0x154AF4F4, 0x229E8F8F, 0x12A2ABAB, 0x584E6262, 0x07E85F5F,
-    0x99E51D1D, 0x34392323, 0x6EC1F6F6, 0x50446C6C, 0xDE5D3232, 0x68724646,
-    0x6526A0A0, 0xBC93CDCD, 0xDB03DADA, 0xF8C6BABA, 0xC8FA9E9E, 0xA882D6D6,
-    0x2BCF6E6E, 0x40507070, 0xDCEB8585, 0xFE750A0A, 0x328A9393, 0xA48DDFDF,
-    0xCA4C2929, 0x10141C1C, 0x2173D7D7, 0xF0CCB4B4, 0xD309D4D4, 0x5D108A8A,
-    0x0FE25151, 0x00000000, 0x6F9A1919, 0x9DE01A1A, 0x368F9494, 0x42E6C7C7,
-    0x4AECC9C9, 0x5EFDD2D2, 0xC1AB7F7F, 0xE0D8A8A8 },
-
-  { 0xBC75BC32, 0xECF3EC21, 0x20C62043, 0xB3F4B3C9, 0xDADBDA03, 0x027B028B,
-    0xE2FBE22B, 0x9EC89EFA, 0xC94AC9EC, 0xD4D3D409, 0x18E6186B, 0x1E6B1E9F,
-    0x9845980E, 0xB27DB238, 0xA6E8A6D2, 0x264B26B7, 0x3CD63C57, 0x9332938A,
-    0x82D882EE, 0x52FD5298, 0x7B377BD4, 0xBB71BB37, 0x5BF15B97, 0x47E14783,
-    0x2430243C, 0x510F51E2, 0xBAF8BAC6, 0x4A1B4AF3, 0xBF87BF48, 0x0DFA0D70,
-    0xB006B0B3, 0x753F75DE, 0xD25ED2FD, 0x7DBA7D20, 0x66AE6631, 0x3A5B3AA3,
-    0x598A591C, 0x00000000, 0xCDBCCD93, 0x1A9D1AE0, 0xAE6DAE2C, 0x7FC17FAB,
-    0x2BB12BC7, 0xBE0EBEB9, 0xE080E0A0, 0x8A5D8A10, 0x3BD23B52, 0x64D564BA,
-    0xD8A0D888, 0xE784E7A5, 0x5F075FE8, 0x1B141B11, 0x2CB52CC2, 0xFC90FCB4,
-    0x312C3127, 0x80A38065, 0x73B2732A, 0x0C730C81, 0x794C795F, 0x6B546B41,
-    0x4B924B02, 0x53745369, 0x9436948F, 0x8351831F, 0x2A382A36, 0xC4B0C49C,
-    0x22BD22C8, 0xD55AD5F8, 0xBDFCBDC3, 0x48604878, 0xFF62FFCE, 0x4C964C07,
-    0x416C4177, 0xC742C7E6, 0xEBF7EB24, 0x1C101C14, 0x5D7C5D63, 0x36283622,
-    0x672767C0, 0xE98CE9AF, 0x441344F9, 0x149514EA, 0xF59CF5BB, 0xCFC7CF18,
-    0x3F243F2D, 0xC046C0E3, 0x723B72DB, 0x5470546C, 0x29CA294C, 0xF0E3F035,
-    0x088508FE, 0xC6CBC617, 0xF311F34F, 0x8CD08CE4, 0xA493A459, 0xCAB8CA96,
-    0x68A6683B, 0xB883B84D, 0x38203828, 0xE5FFE52E, 0xAD9FAD56, 0x0B770B84,
-    0xC8C3C81D, 0x99CC99FF, 0x580358ED, 0x196F199A, 0x0E080E0A, 0x95BF957E,
-    0x70407050, 0xF7E7F730, 0x6E2B6ECF, 0x1FE21F6E, 0xB579B53D, 0x090C090F,
-    0x61AA6134, 0x57825716, 0x9F419F0B, 0x9D3A9D80, 0x11EA1164, 0x25B925CD,
-    0xAFE4AFDD, 0x459A4508, 0xDFA4DF8D, 0xA397A35C, 0xEA7EEAD5, 0x35DA3558,
-    0xED7AEDD0, 0x431743FC, 0xF866F8CB, 0xFB94FBB1, 0x37A137D3, 0xFA1DFA40,
-    0xC23DC268, 0xB4F0B4CC, 0x32DE325D, 0x9CB39C71, 0x560B56E7, 0xE372E3DA,
-    0x87A78760, 0x151C151B, 0xF9EFF93A, 0x63D163BF, 0x345334A9, 0x9A3E9A85,
-    0xB18FB142, 0x7C337CD1, 0x8826889B, 0x3D5F3DA6, 0xA1ECA1D7, 0xE476E4DF,
-    0x812A8194, 0x91499101, 0x0F810FFB, 0xEE88EEAA, 0x16EE1661, 0xD721D773,
-    0x97C497F5, 0xA51AA5A8, 0xFEEBFE3F, 0x6DD96DB5, 0x78C578AE, 0xC539C56D,
-    0x1D991DE5, 0x76CD76A4, 0x3EAD3EDC, 0xCB31CB67, 0xB68BB647, 0xEF01EF5B,
-    0x1218121E, 0x602360C5, 0x6ADD6AB0, 0x4D1F4DF6, 0xCE4ECEE9, 0xDE2DDE7C,
-    0x55F9559D, 0x7E487E5A, 0x214F21B2, 0x03F2037A, 0xA065A026, 0x5E8E5E19,
-    0x5A785A66, 0x655C654B, 0x6258624E, 0xFD19FD45, 0x068D06F4, 0x40E54086,
-    0xF298F2BE, 0x335733AC, 0x17671790, 0x057F058E, 0xE805E85E, 0x4F644F7D,
-    0x89AF896A, 0x10631095, 0x74B6742F, 0x0AFE0A75, 0x5CF55C92, 0x9BB79B74,
-    0x2D3C2D33, 0x30A530D6, 0x2ECE2E49, 0x49E94989, 0x46684672, 0x77447755,
-    0xA8E0A8D8, 0x964D9604, 0x284328BD, 0xA969A929, 0xD929D979, 0x862E8691,
-    0xD1ACD187, 0xF415F44A, 0x8D598D15, 0xD6A8D682, 0xB90AB9BC, 0x429E420D,
-    0xF66EF6C1, 0x2F472FB8, 0xDDDFDD06, 0x23342339, 0xCC35CC62, 0xF16AF1C4,
-    0xC1CFC112, 0x85DC85EB, 0x8F228F9E, 0x71C971A1, 0x90C090F0, 0xAA9BAA53,
-    0x018901F1, 0x8BD48BE1, 0x4EED4E8C, 0x8EAB8E6F, 0xAB12ABA2, 0x6FA26F3E,
-    0xE60DE654, 0xDB52DBF2, 0x92BB927B, 0xB702B7B6, 0x692F69CA, 0x39A939D9,
-    0xD3D7D30C, 0xA761A723, 0xA21EA2AD, 0xC3B4C399, 0x6C506C44, 0x07040705,
-    0x04F6047F, 0x27C22746, 0xAC16ACA7, 0xD025D076, 0x50865013, 0xDC56DCF7,
-    0x8455841A, 0xE109E151, 0x7ABE7A25, 0x139113EF },
-
-  { 0xD939A9D9, 0x90176790, 0x719CB371, 0xD2A6E8D2, 0x05070405, 0x9852FD98,
-    0x6580A365, 0xDFE476DF, 0x08459A08, 0x024B9202, 0xA0E080A0, 0x665A7866,
-    0xDDAFE4DD, 0xB06ADDB0, 0xBF63D1BF, 0x362A3836, 0x54E60D54, 0x4320C643,
-    0x62CC3562, 0xBEF298BE, 0x1E12181E, 0x24EBF724, 0xD7A1ECD7, 0x77416C77,
-    0xBD2843BD, 0x32BC7532, 0xD47B37D4, 0x9B88269B, 0x700DFA70, 0xF94413F9,
-    0xB1FB94B1, 0x5A7E485A, 0x7A03F27A, 0xE48CD0E4, 0x47B68B47, 0x3C24303C,
-    0xA5E784A5, 0x416B5441, 0x06DDDF06, 0xC56023C5, 0x45FD1945, 0xA33A5BA3,
-    0x68C23D68, 0x158D5915, 0x21ECF321, 0x3166AE31, 0x3E6FA23E, 0x16578216,
-    0x95106395, 0x5BEF015B, 0x4DB8834D, 0x91862E91, 0xB56DD9B5, 0x1F83511F,
-    0x53AA9B53, 0x635D7C63, 0x3B68A63B, 0x3FFEEB3F, 0xD630A5D6, 0x257ABE25,
-    0xA7AC16A7, 0x0F090C0F, 0x35F0E335, 0x23A76123, 0xF090C0F0, 0xAFE98CAF,
-    0x809D3A80, 0x925CF592, 0x810C7381, 0x27312C27, 0x76D02576, 0xE7560BE7,
-    0x7B92BB7B, 0xE9CE4EE9, 0xF10189F1, 0x9F1E6B9F, 0xA93453A9, 0xC4F16AC4,
-    0x99C3B499, 0x975BF197, 0x8347E183, 0x6B18E66B, 0xC822BDC8, 0x0E98450E,
-    0x6E1FE26E, 0xC9B3F4C9, 0x2F74B62F, 0xCBF866CB, 0xFF99CCFF, 0xEA1495EA,
-    0xED5803ED, 0xF7DC56F7, 0xE18BD4E1, 0x1B151C1B, 0xADA21EAD, 0x0CD3D70C,
-    0x2BE2FB2B, 0x1DC8C31D, 0x195E8E19, 0xC22CB5C2, 0x8949E989, 0x12C1CF12,
-    0x7E95BF7E, 0x207DBA20, 0x6411EA64, 0x840B7784, 0x6DC5396D, 0x6A89AF6A,
-    0xD17C33D1, 0xA171C9A1, 0xCEFF62CE, 0x37BB7137, 0xFB0F81FB, 0x3DB5793D,
-    0x51E10951, 0xDC3EADDC, 0x2D3F242D, 0xA476CDA4, 0x9D55F99D, 0xEE82D8EE,
-    0x8640E586, 0xAE78C5AE, 0xCD25B9CD, 0x04964D04, 0x55774455, 0x0A0E080A,
-    0x13508613, 0x30F7E730, 0xD337A1D3, 0x40FA1D40, 0x3461AA34, 0x8C4EED8C,
-    0xB3B006B3, 0x6C54706C, 0x2A73B22A, 0x523BD252, 0x0B9F410B, 0x8B027B8B,
-    0x88D8A088, 0x4FF3114F, 0x67CB3167, 0x4627C246, 0xC06727C0, 0xB4FC90B4,
-    0x28382028, 0x7F04F67F, 0x78486078, 0x2EE5FF2E, 0x074C9607, 0x4B655C4B,
-    0xC72BB1C7, 0x6F8EAB6F, 0x0D429E0D, 0xBBF59CBB, 0xF2DB52F2, 0xF34A1BF3,
-    0xA63D5FA6, 0x59A49359, 0xBCB90ABC, 0x3AF9EF3A, 0xEF1391EF, 0xFE0885FE,
-    0x01914901, 0x6116EE61, 0x7CDE2D7C, 0xB2214FB2, 0x42B18F42, 0xDB723BDB,
-    0xB82F47B8, 0x48BF8748, 0x2CAE6D2C, 0xE3C046E3, 0x573CD657, 0x859A3E85,
-    0x29A96929, 0x7D4F647D, 0x94812A94, 0x492ECE49, 0x17C6CB17, 0xCA692FCA,
-    0xC3BDFCC3, 0x5CA3975C, 0x5EE8055E, 0xD0ED7AD0, 0x87D1AC87, 0x8E057F8E,
-    0xBA64D5BA, 0xA8A51AA8, 0xB7264BB7, 0xB9BE0EB9, 0x6087A760, 0xF8D55AF8,
-    0x22362822, 0x111B1411, 0xDE753FDE, 0x79D92979, 0xAAEE88AA, 0x332D3C33,
-    0x5F794C5F, 0xB6B702B6, 0x96CAB896, 0x5835DA58, 0x9CC4B09C, 0xFC4317FC,
-    0x1A84551A, 0xF64D1FF6, 0x1C598A1C, 0x38B27D38, 0xAC3357AC, 0x18CFC718,
-    0xF4068DF4, 0x69537469, 0x749BB774, 0xF597C4F5, 0x56AD9F56, 0xDAE372DA,
-    0xD5EA7ED5, 0x4AF4154A, 0x9E8F229E, 0xA2AB12A2, 0x4E62584E, 0xE85F07E8,
-    0xE51D99E5, 0x39233439, 0xC1F66EC1, 0x446C5044, 0x5D32DE5D, 0x72466872,
-    0x26A06526, 0x93CDBC93, 0x03DADB03, 0xC6BAF8C6, 0xFA9EC8FA, 0x82D6A882,
-    0xCF6E2BCF, 0x50704050, 0xEB85DCEB, 0x750AFE75, 0x8A93328A, 0x8DDFA48D,
-    0x4C29CA4C, 0x141C1014, 0x73D72173, 0xCCB4F0CC, 0x09D4D309, 0x108A5D10,
-    0xE2510FE2, 0x00000000, 0x9A196F9A, 0xE01A9DE0, 0x8F94368F, 0xE6C742E6,
-    0xECC94AEC, 0xFDD25EFD, 0xAB7FC1AB, 0xD8A8E0D8 }
-};
-
-#define mds(n,x) m_tab[n][x]
-
-u32 h_fun (u32 *sk, u32 *lk, const u32 x, const u32 *key)
-{
-  u32  b0, b1, b2, b3;
-
-  b0 = extract_byte (x, 0);
-  b1 = extract_byte (x, 1);
-  b2 = extract_byte (x, 2);
-  b3 = extract_byte (x, 3);
-
-  b0 = q (1, b0) ^ extract_byte (key[3], 0);
-  b1 = q (0, b1) ^ extract_byte (key[3], 1);
-  b2 = q (0, b2) ^ extract_byte (key[3], 2);
-  b3 = q (1, b3) ^ extract_byte (key[3], 3);
-
-  b0 = q (1, b0) ^ extract_byte (key[2], 0);
-  b1 = q (1, b1) ^ extract_byte (key[2], 1);
-  b2 = q (0, b2) ^ extract_byte (key[2], 2);
-  b3 = q (0, b3) ^ extract_byte (key[2], 3);
-
-  b0 = q (0, (q (0, b0) ^ extract_byte (key[1], 0))) ^ extract_byte (key[0], 0);
-  b1 = q (0, (q (1, b1) ^ extract_byte (key[1], 1))) ^ extract_byte (key[0], 1);
-  b2 = q (1, (q (0, b2) ^ extract_byte (key[1], 2))) ^ extract_byte (key[0], 2);
-  b3 = q (1, (q (1, b3) ^ extract_byte (key[1], 3))) ^ extract_byte (key[0], 3);
-
-  return mds (0, b0) ^ mds (1, b1) ^ mds (2, b2) ^ mds (3, b3);
-}
-
-#define q40(x,k) q (0, q (0, q (1, q (1, x) ^ extract_byte (k[3], 0)) ^ extract_byte (k[2], 0)) ^ extract_byte (k[1], 0)) ^ extract_byte (k[0], 0)
-#define q41(x,k) q (0, q (1, q (1, q (0, x) ^ extract_byte (k[3], 1)) ^ extract_byte (k[2], 1)) ^ extract_byte (k[1], 1)) ^ extract_byte (k[0], 1)
-#define q42(x,k) q (1, q (0, q (0, q (0, x) ^ extract_byte (k[3], 2)) ^ extract_byte (k[2], 2)) ^ extract_byte (k[1], 2)) ^ extract_byte (k[0], 2)
-#define q43(x,k) q (1, q (1, q (0, q (1, x) ^ extract_byte (k[3], 3)) ^ extract_byte (k[2], 3)) ^ extract_byte (k[1], 3)) ^ extract_byte (k[0], 3)
-
-#define g1_fun(x)                                   \
-  (mds (0, q40 (extract_byte (x, 3), sk)) ^ \
-   mds (1, q41 (extract_byte (x, 0), sk)) ^ \
-   mds (2, q42 (extract_byte (x, 1), sk)) ^ \
-   mds (3, q43 (extract_byte (x, 2), sk)))
-
-#define g0_fun(x)                                   \
-  (mds (0, q40 (extract_byte (x, 0), sk)) ^ \
-   mds (1, q41 (extract_byte (x, 1), sk)) ^ \
-   mds (2, q42 (extract_byte (x, 2), sk)) ^ \
-   mds (3, q43 (extract_byte (x, 3), sk)))
-
-u32 mds_rem (u32 p0, u32 p1)
-{
-  #define G_MOD 0x14d
-
-  for (int i = 0; i < 8; i++)
-  {
-    u32 t = p1 >> 24;
-
-    p1 = (p1 << 8) | (p0 >> 24);
-
-    p0 <<= 8;
-
-    u32 u = (t << 1);
-
-    if (t & 0x80) u ^= G_MOD;
-
-    p1 ^= t ^ (u << 16);
-
-    u ^= (t >> 1);
-
-    if (t & 0x01) u ^= G_MOD >> 1;
-
-    p1 ^= (u << 24) | (u << 8);
-  }
-
-  return p1;
-}
-
-void twofish256_set_key (u32 *sk, u32 *lk, const u32 *ukey)
-{
-  u32 me_key[4];
-
-  me_key[0] = ukey[0];
-  me_key[1] = ukey[2];
-  me_key[2] = ukey[4];
-  me_key[3] = ukey[6];
-
-  u32 mo_key[4];
-
-  mo_key[0] = ukey[1];
-  mo_key[1] = ukey[3];
-  mo_key[2] = ukey[5];
-  mo_key[3] = ukey[7];
-
-  sk[3] = mds_rem (me_key[0], mo_key[0]);
-  sk[2] = mds_rem (me_key[1], mo_key[1]);
-  sk[1] = mds_rem (me_key[2], mo_key[2]);
-  sk[0] = mds_rem (me_key[3], mo_key[3]);
-
-  for (int i = 0; i < 40; i += 2)
-  {
-    u32 a = 0x01010101 * i;
-    u32 b = 0x01010101 + a;
-
-    a = h_fun (sk, lk, a, me_key);
-    b = h_fun (sk, lk, b, mo_key);
-
-    b = rotl32 (b, 8);
-
-    lk[i + 0] = a + b;
-    lk[i + 1] = rotl32 (a + 2 * b, 9);
-  }
-}
-
-#define f_rnd(i)                                                    \
-{                                                                   \
-  u32 t0 = g0_fun (data[0]);                                       \
-  u32 t1 = g1_fun (data[1]);                                       \
-  data[2] = rotr32 (data[2] ^ (t0 + t1 + lk[4 * (i) + 8]), 1);      \
-  data[3] = rotl32 (data[3], 1) ^ (t0 + 2 * t1 + lk[4 * (i) + 9]);  \
-  u32 t2 = g0_fun (data[2]);                                       \
-  u32 t3 = g1_fun (data[3]);                                       \
-  data[0] = rotr32 (data[0] ^ (t2 + t3 + lk[4 * (i) + 10]), 1);     \
-  data[1] = rotl32 (data[1], 1) ^ (t2 + 2 * t3 + lk[4 * (i) + 11]); \
-}
-
-void twofish256_encrypt (const u32 *sk, const u32 *lk, const u32 *in, u32 *out)
-{
-  u32 data[4];
-
-  data[0] = in[0] ^ lk[0];
-  data[1] = in[1] ^ lk[1];
-  data[2] = in[2] ^ lk[2];
-  data[3] = in[3] ^ lk[3];
-
-  f_rnd (0);
-  f_rnd (1);
-  f_rnd (2);
-  f_rnd (3);
-  f_rnd (4);
-  f_rnd (5);
-  f_rnd (6);
-  f_rnd (7);
-
-  out[0] = data[2] ^ lk[4];
-  out[1] = data[3] ^ lk[5];
-  out[2] = data[0] ^ lk[6];
-  out[3] = data[1] ^ lk[7];
-}
-
-#define i_rnd(i)                                                    \
-{                                                                   \
-  u32 t0 = g0_fun (data[0]);                                       \
-  u32 t1 = g1_fun (data[1]);                                       \
-  data[2] = rotl32 (data[2], 1) ^ (t0 + t1 + lk[4 * (i) + 10]);     \
-  data[3] = rotr32 (data[3] ^ (t0 + 2 * t1 + lk[4 * (i) + 11]), 1); \
-  u32 t2 = g0_fun (data[2]);                                       \
-  u32 t3 = g1_fun (data[3]);                                       \
-  data[0] = rotl32 (data[0], 1) ^ (t2 + t3 + lk[4 * (i) +  8]);     \
-  data[1] = rotr32 (data[1] ^ (t2 + 2 * t3 + lk[4 * (i) +  9]), 1); \
-}
-
-void twofish256_decrypt (const u32 *sk, const u32 *lk, const u32 *in, u32 *out)
-{
-  u32 data[4];
-
-  data[0] = in[0] ^ lk[4];
-  data[1] = in[1] ^ lk[5];
-  data[2] = in[2] ^ lk[6];
-  data[3] = in[3] ^ lk[7];
-
-  i_rnd (7);
-  i_rnd (6);
-  i_rnd (5);
-  i_rnd (4);
-  i_rnd (3);
-  i_rnd (2);
-  i_rnd (1);
-  i_rnd (0);
-
-  out[0] = data[2] ^ lk[0];
-  out[1] = data[3] ^ lk[1];
-  out[2] = data[0] ^ lk[2];
-  out[3] = data[1] ^ lk[3];
-}
-
-void twofish256_decrypt_xts (const u32 *ukey1, const u32 *ukey2, const u32 *in, u32 *out)
-{
-  u32 T[4] = { 0 };
-  u32 Z[4] = { 0 };
-
-  out[0] = in[0];
-  out[1] = in[1];
-  out[2] = in[2];
-  out[3] = in[3];
-
-  u32 sk[4]; u32 lk[40];
-
-  twofish256_set_key (sk, lk, ukey2);
-  twofish256_encrypt (sk, lk, Z, T);
-
-  out[0] ^= T[0];
-  out[1] ^= T[1];
-  out[2] ^= T[2];
-  out[3] ^= T[3];
-
-  twofish256_set_key (sk, lk, ukey1);
-  twofish256_decrypt (sk, lk, out, out);
-
-  out[0] ^= T[0];
-  out[1] ^= T[1];
-  out[2] ^= T[2];
-  out[3] ^= T[3];
-}

diff --git a/OpenCL/m00000_a0.cl b/OpenCL/m00000_a0.cl
index 08e5923..cdc550a 100644 (file)
--- a/OpenCL/m00000_a0.cl
+++ b/OpenCL/m00000_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00000_a1.cl b/OpenCL/m00000_a1.cl
index c89d6f2..a5df408 100644 (file)
--- a/OpenCL/m00000_a1.cl
+++ b/OpenCL/m00000_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00000_a3.cl b/OpenCL/m00000_a3.cl
index 09b4665..e4d86bd 100644 (file)
--- a/OpenCL/m00000_a3.cl
+++ b/OpenCL/m00000_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define MD5_STEP_REV(f,a,b,c,d,x,t,s)   \
 {                                       \

diff --git a/OpenCL/m00010_a0.cl b/OpenCL/m00010_a0.cl
index d2beb72..416dc30 100644 (file)
--- a/OpenCL/m00010_a0.cl
+++ b/OpenCL/m00010_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00010_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00010_a1.cl b/OpenCL/m00010_a1.cl
index a1424c9..e3ffd36 100644 (file)
--- a/OpenCL/m00010_a1.cl
+++ b/OpenCL/m00010_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00010_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00010_a3.cl b/OpenCL/m00010_a3.cl
index 38ef98f..d923bd3 100644 (file)
--- a/OpenCL/m00010_a3.cl
+++ b/OpenCL/m00010_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define MD5_STEP_REV(f,a,b,c,d,x,t,s)   \
 {                                       \

diff --git a/OpenCL/m00020_a0.cl b/OpenCL/m00020_a0.cl
index 99fc13a..db3d526 100644 (file)
--- a/OpenCL/m00020_a0.cl
+++ b/OpenCL/m00020_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00020_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00020_a1.cl b/OpenCL/m00020_a1.cl
index 3e5d995..8318aa2 100644 (file)
--- a/OpenCL/m00020_a1.cl
+++ b/OpenCL/m00020_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00020_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00020_a3.cl b/OpenCL/m00020_a3.cl
index 13d0651..d4098f9 100644 (file)
--- a/OpenCL/m00020_a3.cl
+++ b/OpenCL/m00020_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m00020m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m00030_a0.cl b/OpenCL/m00030_a0.cl
index 9309e30..cd86c4c 100644 (file)
--- a/OpenCL/m00030_a0.cl
+++ b/OpenCL/m00030_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 // no unicode yet
 

diff --git a/OpenCL/m00030_a1.cl b/OpenCL/m00030_a1.cl
index 329c8c1..3ae93f4 100644 (file)
--- a/OpenCL/m00030_a1.cl
+++ b/OpenCL/m00030_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 // no unicode yet
 

diff --git a/OpenCL/m00030_a3.cl b/OpenCL/m00030_a3.cl
index 05001ed..4b26384 100644 (file)
--- a/OpenCL/m00030_a3.cl
+++ b/OpenCL/m00030_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define MD5_STEP_REV(f,a,b,c,d,x,t,s)   \
 {                                       \

diff --git a/OpenCL/m00040_a0.cl b/OpenCL/m00040_a0.cl
index f888f52..9521b8d 100644 (file)
--- a/OpenCL/m00040_a0.cl
+++ b/OpenCL/m00040_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00040_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00040_a1.cl b/OpenCL/m00040_a1.cl
index 2996c90..49c6046 100644 (file)
--- a/OpenCL/m00040_a1.cl
+++ b/OpenCL/m00040_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 // no unicode yet
 

diff --git a/OpenCL/m00040_a3.cl b/OpenCL/m00040_a3.cl
index ee4ef2e..abb8c30 100644 (file)
--- a/OpenCL/m00040_a3.cl
+++ b/OpenCL/m00040_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m00040m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m00050_a0.cl b/OpenCL/m00050_a0.cl
index eb34668..9fc1533 100644 (file)
--- a/OpenCL/m00050_a0.cl
+++ b/OpenCL/m00050_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m00050_a1.cl b/OpenCL/m00050_a1.cl
index ec578f1..b269ad0 100644 (file)
--- a/OpenCL/m00050_a1.cl
+++ b/OpenCL/m00050_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m00050_a3.cl b/OpenCL/m00050_a3.cl
index 1e93f46..f08226e 100644 (file)
--- a/OpenCL/m00050_a3.cl
+++ b/OpenCL/m00050_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m00060_a0.cl b/OpenCL/m00060_a0.cl
index 687738c..e2eeddb 100644 (file)
--- a/OpenCL/m00060_a0.cl
+++ b/OpenCL/m00060_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m00060_a1.cl b/OpenCL/m00060_a1.cl
index 85604b8..635b081 100644 (file)
--- a/OpenCL/m00060_a1.cl
+++ b/OpenCL/m00060_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m00060_a3.cl b/OpenCL/m00060_a3.cl
index 2863aef..fc636d2 100644 (file)
--- a/OpenCL/m00060_a3.cl
+++ b/OpenCL/m00060_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m00100_a0.cl b/OpenCL/m00100_a0.cl
index 4faa35e..7cc5b2b 100644 (file)
--- a/OpenCL/m00100_a0.cl
+++ b/OpenCL/m00100_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00100_a1.cl b/OpenCL/m00100_a1.cl
index 51be262..39c2177 100644 (file)
--- a/OpenCL/m00100_a1.cl
+++ b/OpenCL/m00100_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00100_a3.cl b/OpenCL/m00100_a3.cl
index c78f744..455b840 100644 (file)
--- a/OpenCL/m00100_a3.cl
+++ b/OpenCL/m00100_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m00100m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m00110_a0.cl b/OpenCL/m00110_a0.cl
index e8ce8d9..09dac89 100644 (file)
--- a/OpenCL/m00110_a0.cl
+++ b/OpenCL/m00110_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00110_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00110_a1.cl b/OpenCL/m00110_a1.cl
index af0af26..c92ed52 100644 (file)
--- a/OpenCL/m00110_a1.cl
+++ b/OpenCL/m00110_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00110_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00110_a3.cl b/OpenCL/m00110_a3.cl
index b5d3fb0..b316a68 100644 (file)
--- a/OpenCL/m00110_a3.cl
+++ b/OpenCL/m00110_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m00110m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m00120_a0.cl b/OpenCL/m00120_a0.cl
index 61286b5..0123ccc 100644 (file)
--- a/OpenCL/m00120_a0.cl
+++ b/OpenCL/m00120_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00120_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00120_a1.cl b/OpenCL/m00120_a1.cl
index 1c282b9..a31c629 100644 (file)
--- a/OpenCL/m00120_a1.cl
+++ b/OpenCL/m00120_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00120_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00120_a3.cl b/OpenCL/m00120_a3.cl
index 98f8c02..080c497 100644 (file)
--- a/OpenCL/m00120_a3.cl
+++ b/OpenCL/m00120_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m00120m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m00130_a0.cl b/OpenCL/m00130_a0.cl
index c62f198..a92db01 100644 (file)
--- a/OpenCL/m00130_a0.cl
+++ b/OpenCL/m00130_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 // no unicode yet
 

diff --git a/OpenCL/m00130_a1.cl b/OpenCL/m00130_a1.cl
index d766c06..6f216c5 100644 (file)
--- a/OpenCL/m00130_a1.cl
+++ b/OpenCL/m00130_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 // no unicode yet
 

diff --git a/OpenCL/m00130_a3.cl b/OpenCL/m00130_a3.cl
index a9689b0..96fe640 100644 (file)
--- a/OpenCL/m00130_a3.cl
+++ b/OpenCL/m00130_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m00130m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m00140_a0.cl b/OpenCL/m00140_a0.cl
index f92ccb6..066f9bf 100644 (file)
--- a/OpenCL/m00140_a0.cl
+++ b/OpenCL/m00140_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 // no unicode yet
 

diff --git a/OpenCL/m00140_a1.cl b/OpenCL/m00140_a1.cl
index 50cef28..36a0f92 100644 (file)
--- a/OpenCL/m00140_a1.cl
+++ b/OpenCL/m00140_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 // no unicode yet
 

diff --git a/OpenCL/m00140_a3.cl b/OpenCL/m00140_a3.cl
index 0dd5758..3439437 100644 (file)
--- a/OpenCL/m00140_a3.cl
+++ b/OpenCL/m00140_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m00140m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m00150_a0.cl b/OpenCL/m00150_a0.cl
index 68df4b4..022208a 100644 (file)
--- a/OpenCL/m00150_a0.cl
+++ b/OpenCL/m00150_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m00150_a1.cl b/OpenCL/m00150_a1.cl
index 22ecebb..9a0e150 100644 (file)
--- a/OpenCL/m00150_a1.cl
+++ b/OpenCL/m00150_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m00150_a3.cl b/OpenCL/m00150_a3.cl
index 1e9c5ec..35d1790 100644 (file)
--- a/OpenCL/m00150_a3.cl
+++ b/OpenCL/m00150_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m00160_a0.cl b/OpenCL/m00160_a0.cl
index e5d8bcf..a3c0813 100644 (file)
--- a/OpenCL/m00160_a0.cl
+++ b/OpenCL/m00160_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m00160_a1.cl b/OpenCL/m00160_a1.cl
index 8427e86..370a7a0 100644 (file)
--- a/OpenCL/m00160_a1.cl
+++ b/OpenCL/m00160_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m00160_a3.cl b/OpenCL/m00160_a3.cl
index c7afc3f..4bb85c7 100644 (file)
--- a/OpenCL/m00160_a3.cl
+++ b/OpenCL/m00160_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m00190_a0.cl b/OpenCL/m00190_a0.cl
index 2929228..bc88d6f 100644 (file)
--- a/OpenCL/m00190_a0.cl
+++ b/OpenCL/m00190_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 4
 #define DGST_R2 3
 #define DGST_R3 2
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00190_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00190_a1.cl b/OpenCL/m00190_a1.cl
index db57ca0..ac60807 100644 (file)
--- a/OpenCL/m00190_a1.cl
+++ b/OpenCL/m00190_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 4
 #define DGST_R2 3
 #define DGST_R3 2
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00190_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00190_a3.cl b/OpenCL/m00190_a3.cl
index 15716ed..61d8ece 100644 (file)
--- a/OpenCL/m00190_a3.cl
+++ b/OpenCL/m00190_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 4
 #define DGST_R2 3
 #define DGST_R3 2
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m00190m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m00200_a0.cl b/OpenCL/m00200_a0.cl
index 8f58b40..febb350 100644 (file)
--- a/OpenCL/m00200_a0.cl
+++ b/OpenCL/m00200_a0.cl
@@ -8,20 +8,20 @@
 //incompatible
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00200_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00200_a1.cl b/OpenCL/m00200_a1.cl
index 8ce0b86..390d182 100644 (file)
--- a/OpenCL/m00200_a1.cl
+++ b/OpenCL/m00200_a1.cl
@@ -8,18 +8,18 @@
 //incompatible
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00200_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00200_a3.cl b/OpenCL/m00200_a3.cl
index 27a10c5..7605dd8 100644 (file)
--- a/OpenCL/m00200_a3.cl
+++ b/OpenCL/m00200_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define ROUND(v)                              \
 {                                             \

diff --git a/OpenCL/m00300_a0.cl b/OpenCL/m00300_a0.cl
index c083b02..c202bbc 100644 (file)
--- a/OpenCL/m00300_a0.cl
+++ b/OpenCL/m00300_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00300_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00300_a1.cl b/OpenCL/m00300_a1.cl
index 2af68fe..367751e 100644 (file)
--- a/OpenCL/m00300_a1.cl
+++ b/OpenCL/m00300_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00300_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00300_a3.cl b/OpenCL/m00300_a3.cl
index b722c7a..0f1cfce 100644 (file)
--- a/OpenCL/m00300_a3.cl
+++ b/OpenCL/m00300_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m00300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m00400.cl b/OpenCL/m00400.cl
index d8edbcb..c220977 100644 (file)
--- a/OpenCL/m00400.cl
+++ b/OpenCL/m00400.cl
@@ -7,21 +7,21 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 void md5_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4])
 {

diff --git a/OpenCL/m00500.cl b/OpenCL/m00500.cl
index 6fbc6d3..462e4ee 100644 (file)
--- a/OpenCL/m00500.cl
+++ b/OpenCL/m00500.cl
@@ -5,20 +5,20 @@
 
 #define _MD5_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 #define md5crypt_magic 0x00243124u
 

diff --git a/OpenCL/m00900_a0.cl b/OpenCL/m00900_a0.cl
index 376b83e..2cb123c 100644 (file)
--- a/OpenCL/m00900_a0.cl
+++ b/OpenCL/m00900_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m00900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00900_a1.cl b/OpenCL/m00900_a1.cl
index d7d26f5..26b2119 100644 (file)
--- a/OpenCL/m00900_a1.cl
+++ b/OpenCL/m00900_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m00900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m00900_a3.cl b/OpenCL/m00900_a3.cl
index 5170c0e..2bbf306 100644 (file)
--- a/OpenCL/m00900_a3.cl
+++ b/OpenCL/m00900_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define MD4_STEP_REV(f,a,b,c,d,x,t,s)   \
 {                                       \

diff --git a/OpenCL/m01000_a0.cl b/OpenCL/m01000_a0.cl
index 70fc825..1355f15 100644 (file)
--- a/OpenCL/m01000_a0.cl
+++ b/OpenCL/m01000_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m01000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01000_a1.cl b/OpenCL/m01000_a1.cl
index d5455aa..2c0fcf0 100644 (file)
--- a/OpenCL/m01000_a1.cl
+++ b/OpenCL/m01000_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m01000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01000_a3.cl b/OpenCL/m01000_a3.cl
index f8de4c9..02dffee 100644 (file)
--- a/OpenCL/m01000_a3.cl
+++ b/OpenCL/m01000_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define MD4_STEP_REV(f,a,b,c,d,x,t,s)   \
 {                                       \

diff --git a/OpenCL/m01100_a0.cl b/OpenCL/m01100_a0.cl
index 1cdce45..e13d20f 100644 (file)
--- a/OpenCL/m01100_a0.cl
+++ b/OpenCL/m01100_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m01100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01100_a1.cl b/OpenCL/m01100_a1.cl
index ebde581..fbaba9b 100644 (file)
--- a/OpenCL/m01100_a1.cl
+++ b/OpenCL/m01100_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m01100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01100_a3.cl b/OpenCL/m01100_a3.cl
index b1f010a..53d62d1 100644 (file)
--- a/OpenCL/m01100_a3.cl
+++ b/OpenCL/m01100_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m01100m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m01400_a0.cl b/OpenCL/m01400_a0.cl
index 32f694b..3c26ad6 100644 (file)
--- a/OpenCL/m01400_a0.cl
+++ b/OpenCL/m01400_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m01400_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01400_a1.cl b/OpenCL/m01400_a1.cl
index cc4285a..392ebf3 100644 (file)
--- a/OpenCL/m01400_a1.cl
+++ b/OpenCL/m01400_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m01400_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01400_a3.cl b/OpenCL/m01400_a3.cl
index 2e6b3a4..da17856 100644 (file)
--- a/OpenCL/m01400_a3.cl
+++ b/OpenCL/m01400_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m01400m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m01410_a0.cl b/OpenCL/m01410_a0.cl
index 2da15b8..1eb855d 100644 (file)
--- a/OpenCL/m01410_a0.cl
+++ b/OpenCL/m01410_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m01410_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01410_a1.cl b/OpenCL/m01410_a1.cl
index 751be42..ef9a0ce 100644 (file)
--- a/OpenCL/m01410_a1.cl
+++ b/OpenCL/m01410_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m01410_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01410_a3.cl b/OpenCL/m01410_a3.cl
index 13e9027..bfbc8ad 100644 (file)
--- a/OpenCL/m01410_a3.cl
+++ b/OpenCL/m01410_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m01410m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m01420_a0.cl b/OpenCL/m01420_a0.cl
index 5676003..13406c7 100644 (file)
--- a/OpenCL/m01420_a0.cl
+++ b/OpenCL/m01420_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m01420_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01420_a1.cl b/OpenCL/m01420_a1.cl
index d0a0ace..0bfed86 100644 (file)
--- a/OpenCL/m01420_a1.cl
+++ b/OpenCL/m01420_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m01420_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01420_a3.cl b/OpenCL/m01420_a3.cl
index a95d0c2..4f55994 100644 (file)
--- a/OpenCL/m01420_a3.cl
+++ b/OpenCL/m01420_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m01420m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m01430_a0.cl b/OpenCL/m01430_a0.cl
index 905beed..faa387e 100644 (file)
--- a/OpenCL/m01430_a0.cl
+++ b/OpenCL/m01430_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m01430_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01430_a1.cl b/OpenCL/m01430_a1.cl
index 897878f..d74da25 100644 (file)
--- a/OpenCL/m01430_a1.cl
+++ b/OpenCL/m01430_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m01430_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01430_a3.cl b/OpenCL/m01430_a3.cl
index a2cd4a5..cca9aa6 100644 (file)
--- a/OpenCL/m01430_a3.cl
+++ b/OpenCL/m01430_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m01430m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m01440_a0.cl b/OpenCL/m01440_a0.cl
index 7e38628..30b7c8a 100644 (file)
--- a/OpenCL/m01440_a0.cl
+++ b/OpenCL/m01440_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m01440_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01440_a1.cl b/OpenCL/m01440_a1.cl
index 8cf5364..40944ba 100644 (file)
--- a/OpenCL/m01440_a1.cl
+++ b/OpenCL/m01440_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m01440_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m01440_a3.cl b/OpenCL/m01440_a3.cl
index 69363a2..3e945af 100644 (file)
--- a/OpenCL/m01440_a3.cl
+++ b/OpenCL/m01440_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m01440m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m01450_a0.cl b/OpenCL/m01450_a0.cl
index 7e12d30..4b5a534 100644 (file)
--- a/OpenCL/m01450_a0.cl
+++ b/OpenCL/m01450_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m01450_a1.cl b/OpenCL/m01450_a1.cl
index ab683cf..207b8e0 100644 (file)
--- a/OpenCL/m01450_a1.cl
+++ b/OpenCL/m01450_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m01450_a3.cl b/OpenCL/m01450_a3.cl
index 6756571..2f2ac61 100644 (file)
--- a/OpenCL/m01450_a3.cl
+++ b/OpenCL/m01450_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m01460_a0.cl b/OpenCL/m01460_a0.cl
index 3824c2c..6f973a3 100644 (file)
--- a/OpenCL/m01460_a0.cl
+++ b/OpenCL/m01460_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m01460_a1.cl b/OpenCL/m01460_a1.cl
index 6081ef6..042753e 100644 (file)
--- a/OpenCL/m01460_a1.cl
+++ b/OpenCL/m01460_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m01460_a3.cl b/OpenCL/m01460_a3.cl
index 64c0df5..2ddda1b 100644 (file)
--- a/OpenCL/m01460_a3.cl
+++ b/OpenCL/m01460_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m01500_a0.cl b/OpenCL/m01500_a0.cl
index 0007970..041b35b 100644 (file)
--- a/OpenCL/m01500_a0.cl
+++ b/OpenCL/m01500_a0.cl
@@ -10,20 +10,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m01500_a1.cl b/OpenCL/m01500_a1.cl
index 135cf96..7cdc7fb 100644 (file)
--- a/OpenCL/m01500_a1.cl
+++ b/OpenCL/m01500_a1.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m01500_a3.cl b/OpenCL/m01500_a3.cl
index 2585829..2d2e94e 100644 (file)
--- a/OpenCL/m01500_a3.cl
+++ b/OpenCL/m01500_a3.cl
@@ -7,20 +7,20 @@
 
 #define _DES_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4_bs.c"
-#define COMPARE_M "OpenCL/check_multi_comp4_bs.c"
+#define COMPARE_S "inc_comp_single_bs.cl"
+#define COMPARE_M "inc_comp_multi_bs.cl"
 
 #define myselx(a,b,c) ((c) ? (b) : (a))
 

diff --git a/OpenCL/m01600.cl b/OpenCL/m01600.cl
index 8389cee..1b33d2a 100644 (file)
--- a/OpenCL/m01600.cl
+++ b/OpenCL/m01600.cl
@@ -5,20 +5,20 @@
 
 #define _MD5_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 #define md5apr1_magic0 0x72706124u
 #define md5apr1_magic1 0x00002431u

diff --git a/OpenCL/m01700_a0.cl b/OpenCL/m01700_a0.cl
index 5324cb0..c37ed85 100644 (file)
--- a/OpenCL/m01700_a0.cl
+++ b/OpenCL/m01700_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01700_a1.cl b/OpenCL/m01700_a1.cl
index 57adc4f..94486f4 100644 (file)
--- a/OpenCL/m01700_a1.cl
+++ b/OpenCL/m01700_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01700_a3.cl b/OpenCL/m01700_a3.cl
index c7df44e..55f94f6 100644 (file)
--- a/OpenCL/m01700_a3.cl
+++ b/OpenCL/m01700_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01710_a0.cl b/OpenCL/m01710_a0.cl
index e2172f0..7bc0941 100644 (file)
--- a/OpenCL/m01710_a0.cl
+++ b/OpenCL/m01710_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01710_a1.cl b/OpenCL/m01710_a1.cl
index 794c00f..6ea7861 100644 (file)
--- a/OpenCL/m01710_a1.cl
+++ b/OpenCL/m01710_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01710_a3.cl b/OpenCL/m01710_a3.cl
index 794b9b3..44309b7 100644 (file)
--- a/OpenCL/m01710_a3.cl
+++ b/OpenCL/m01710_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01720_a0.cl b/OpenCL/m01720_a0.cl
index 52eb80c..a065011 100644 (file)
--- a/OpenCL/m01720_a0.cl
+++ b/OpenCL/m01720_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01720_a1.cl b/OpenCL/m01720_a1.cl
index 0ed7984..9e721e4 100644 (file)
--- a/OpenCL/m01720_a1.cl
+++ b/OpenCL/m01720_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01720_a3.cl b/OpenCL/m01720_a3.cl
index 12c1552..b2f5b9d 100644 (file)
--- a/OpenCL/m01720_a3.cl
+++ b/OpenCL/m01720_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01730_a0.cl b/OpenCL/m01730_a0.cl
index 2dd7473..7abf875 100644 (file)
--- a/OpenCL/m01730_a0.cl
+++ b/OpenCL/m01730_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01730_a1.cl b/OpenCL/m01730_a1.cl
index e2a0557..d6c8fe5 100644 (file)
--- a/OpenCL/m01730_a1.cl
+++ b/OpenCL/m01730_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01730_a3.cl b/OpenCL/m01730_a3.cl
index 517822b..0506776 100644 (file)
--- a/OpenCL/m01730_a3.cl
+++ b/OpenCL/m01730_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01740_a0.cl b/OpenCL/m01740_a0.cl
index d6574ce..98d49fe 100644 (file)
--- a/OpenCL/m01740_a0.cl
+++ b/OpenCL/m01740_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01740_a1.cl b/OpenCL/m01740_a1.cl
index c859c41..cd3c8af 100644 (file)
--- a/OpenCL/m01740_a1.cl
+++ b/OpenCL/m01740_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01740_a3.cl b/OpenCL/m01740_a3.cl
index 97a98d3..84d0c72 100644 (file)
--- a/OpenCL/m01740_a3.cl
+++ b/OpenCL/m01740_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01750_a0.cl b/OpenCL/m01750_a0.cl
index 4ba5ea2..c72482b 100644 (file)
--- a/OpenCL/m01750_a0.cl
+++ b/OpenCL/m01750_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01750_a1.cl b/OpenCL/m01750_a1.cl
index 65112a3..b35a789 100644 (file)
--- a/OpenCL/m01750_a1.cl
+++ b/OpenCL/m01750_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01750_a3.cl b/OpenCL/m01750_a3.cl
index aff816f..3de7878 100644 (file)
--- a/OpenCL/m01750_a3.cl
+++ b/OpenCL/m01750_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01760_a0.cl b/OpenCL/m01760_a0.cl
index 8883d47..ed2c409 100644 (file)
--- a/OpenCL/m01760_a0.cl
+++ b/OpenCL/m01760_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01760_a1.cl b/OpenCL/m01760_a1.cl
index 1d7895f..eaa8624 100644 (file)
--- a/OpenCL/m01760_a1.cl
+++ b/OpenCL/m01760_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01760_a3.cl b/OpenCL/m01760_a3.cl
index 61bb6d7..5bf5380 100644 (file)
--- a/OpenCL/m01760_a3.cl
+++ b/OpenCL/m01760_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 14
 #define DGST_R1 15
 #define DGST_R2 6
 #define DGST_R3 7
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m01800.cl b/OpenCL/m01800.cl
index 21b1685..4af9eb6 100644 (file)
--- a/OpenCL/m01800.cl
+++ b/OpenCL/m01800.cl
@@ -5,20 +5,20 @@
 
 #define _SHA512_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 // Buggy drivers...
 

diff --git a/OpenCL/m02100.cl b/OpenCL/m02100.cl
index 2e2a8d2..89e45f8 100644 (file)
--- a/OpenCL/m02100.cl
+++ b/OpenCL/m02100.cl
@@ -7,21 +7,21 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 void md4_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4])
 {

diff --git a/OpenCL/m02400_a0.cl b/OpenCL/m02400_a0.cl
index 0d8dc34..592c3e9 100644 (file)
--- a/OpenCL/m02400_a0.cl
+++ b/OpenCL/m02400_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m02400_m04 (__global pw_t *pws, __global kernel_rule_t *  rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m02400_a1.cl b/OpenCL/m02400_a1.cl
index f434286..452ddb7 100644 (file)
--- a/OpenCL/m02400_a1.cl
+++ b/OpenCL/m02400_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m02400_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m02400_a3.cl b/OpenCL/m02400_a3.cl
index 243bd9c..937bd30 100644 (file)
--- a/OpenCL/m02400_a3.cl
+++ b/OpenCL/m02400_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m02400m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m02410_a0.cl b/OpenCL/m02410_a0.cl
index bb1ce35..c9d37a8 100644 (file)
--- a/OpenCL/m02410_a0.cl
+++ b/OpenCL/m02410_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m02410_m04 (__global pw_t *pws, __global kernel_rule_t *  rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m02410_a1.cl b/OpenCL/m02410_a1.cl
index a6b0a59..4e3974f 100644 (file)
--- a/OpenCL/m02410_a1.cl
+++ b/OpenCL/m02410_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m02410_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m02410_a3.cl b/OpenCL/m02410_a3.cl
index d1e0679..4aedaef 100644 (file)
--- a/OpenCL/m02410_a3.cl
+++ b/OpenCL/m02410_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m02410m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m02500.cl b/OpenCL/m02500.cl
index 78999f6..9f90c54 100644 (file)
--- a/OpenCL/m02500.cl
+++ b/OpenCL/m02500.cl
@@ -7,21 +7,21 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 void md5_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4])
 {

diff --git a/OpenCL/m02610_a0.cl b/OpenCL/m02610_a0.cl
index 57c7afc..ab4eaf1 100644 (file)
--- a/OpenCL/m02610_a0.cl
+++ b/OpenCL/m02610_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m02610_a1.cl b/OpenCL/m02610_a1.cl
index 54e90e9..eac506d 100644 (file)
--- a/OpenCL/m02610_a1.cl
+++ b/OpenCL/m02610_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m02610_a3.cl b/OpenCL/m02610_a3.cl
index a7a3c81..677d69c 100644 (file)
--- a/OpenCL/m02610_a3.cl
+++ b/OpenCL/m02610_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m02710_a0.cl b/OpenCL/m02710_a0.cl
index c3ca3e3..3fbf245 100644 (file)
--- a/OpenCL/m02710_a0.cl
+++ b/OpenCL/m02710_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m02710_a1.cl b/OpenCL/m02710_a1.cl
index 8748ac4..37a617b 100644 (file)
--- a/OpenCL/m02710_a1.cl
+++ b/OpenCL/m02710_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m02710_a3.cl b/OpenCL/m02710_a3.cl
index 23b3a25..dd73e7c 100644 (file)
--- a/OpenCL/m02710_a3.cl
+++ b/OpenCL/m02710_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m02810_a0.cl b/OpenCL/m02810_a0.cl
index 5cdd1e6..c9458fe 100644 (file)
--- a/OpenCL/m02810_a0.cl
+++ b/OpenCL/m02810_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m02810_a1.cl b/OpenCL/m02810_a1.cl
index 768f535..19ea018 100644 (file)
--- a/OpenCL/m02810_a1.cl
+++ b/OpenCL/m02810_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m02810_a3.cl b/OpenCL/m02810_a3.cl
index 6ec3014..cdcce5e 100644 (file)
--- a/OpenCL/m02810_a3.cl
+++ b/OpenCL/m02810_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m03000_a0.cl b/OpenCL/m03000_a0.cl
index dc3653d..10ad91a 100644 (file)
--- a/OpenCL/m03000_a0.cl
+++ b/OpenCL/m03000_a0.cl
@@ -10,20 +10,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m03000_a1.cl b/OpenCL/m03000_a1.cl
index 22b6de6..d500e0a 100644 (file)
--- a/OpenCL/m03000_a1.cl
+++ b/OpenCL/m03000_a1.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m03000_a3.cl b/OpenCL/m03000_a3.cl
index f5d3380..cf501f1 100644 (file)
--- a/OpenCL/m03000_a3.cl
+++ b/OpenCL/m03000_a3.cl
@@ -6,20 +6,20 @@
 
 #define _DES_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4_bs.c"
-#define COMPARE_M "OpenCL/check_multi_comp4_bs.c"
+#define COMPARE_S "inc_comp_single_bs.cl"
+#define COMPARE_M "inc_comp_multi_bs.cl"
 
 #ifdef IS_NV
 #define KXX_DECL

diff --git a/OpenCL/m03100_a0.cl b/OpenCL/m03100_a0.cl
index 5f98fda..df52606 100644 (file)
--- a/OpenCL/m03100_a0.cl
+++ b/OpenCL/m03100_a0.cl
@@ -11,20 +11,20 @@
 //incompatible
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m03100_a1.cl b/OpenCL/m03100_a1.cl
index b16db6c..18d7741 100644 (file)
--- a/OpenCL/m03100_a1.cl
+++ b/OpenCL/m03100_a1.cl
@@ -10,18 +10,18 @@
 //incompatible
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m03100_a3.cl b/OpenCL/m03100_a3.cl
index e1729e3..5c25677 100644 (file)
--- a/OpenCL/m03100_a3.cl
+++ b/OpenCL/m03100_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m03200.cl b/OpenCL/m03200.cl
index a3789cd..a278122 100644 (file)
--- a/OpenCL/m03200.cl
+++ b/OpenCL/m03200.cl
@@ -5,20 +5,20 @@
 
 #define _BCRYPT_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 // http://www.schneier.com/code/constants.txt
 

diff --git a/OpenCL/m03710_a0.cl b/OpenCL/m03710_a0.cl
index 4ee6320..74aa5e8 100644 (file)
--- a/OpenCL/m03710_a0.cl
+++ b/OpenCL/m03710_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m03710_a1.cl b/OpenCL/m03710_a1.cl
index f3f777a..c164fdf 100644 (file)
--- a/OpenCL/m03710_a1.cl
+++ b/OpenCL/m03710_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m03710_a3.cl b/OpenCL/m03710_a3.cl
index 7d3a1ab..9e5eacd 100644 (file)
--- a/OpenCL/m03710_a3.cl
+++ b/OpenCL/m03710_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m03800_a0.cl b/OpenCL/m03800_a0.cl
index f4b311e..ee926d8 100644 (file)
--- a/OpenCL/m03800_a0.cl
+++ b/OpenCL/m03800_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m03800_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m03800_a1.cl b/OpenCL/m03800_a1.cl
index e860d07..f727a02 100644 (file)
--- a/OpenCL/m03800_a1.cl
+++ b/OpenCL/m03800_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m03800_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m03800_a3.cl b/OpenCL/m03800_a3.cl
index 9b31135..79af82b 100644 (file)
--- a/OpenCL/m03800_a3.cl
+++ b/OpenCL/m03800_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m03800m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m04310_a0.cl b/OpenCL/m04310_a0.cl
index 0cfd0cd..038f50d 100644 (file)
--- a/OpenCL/m04310_a0.cl
+++ b/OpenCL/m04310_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04310_a1.cl b/OpenCL/m04310_a1.cl
index f4cb7c1..356fd0e 100644 (file)
--- a/OpenCL/m04310_a1.cl
+++ b/OpenCL/m04310_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04310_a3.cl b/OpenCL/m04310_a3.cl
index 5b50455..25c2a75 100644 (file)
--- a/OpenCL/m04310_a3.cl
+++ b/OpenCL/m04310_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04400_a0.cl b/OpenCL/m04400_a0.cl
index f2b7f76..e6f65a4 100644 (file)
--- a/OpenCL/m04400_a0.cl
+++ b/OpenCL/m04400_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04400_a1.cl b/OpenCL/m04400_a1.cl
index 435e041..64a727b 100644 (file)
--- a/OpenCL/m04400_a1.cl
+++ b/OpenCL/m04400_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04400_a3.cl b/OpenCL/m04400_a3.cl
index 76fb33b..3c587c6 100644 (file)
--- a/OpenCL/m04400_a3.cl
+++ b/OpenCL/m04400_a3.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04500_a0.cl b/OpenCL/m04500_a0.cl
index 56076a4..7e71655 100644 (file)
--- a/OpenCL/m04500_a0.cl
+++ b/OpenCL/m04500_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04500_a1.cl b/OpenCL/m04500_a1.cl
index 4fe75a1..d58956b 100644 (file)
--- a/OpenCL/m04500_a1.cl
+++ b/OpenCL/m04500_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04500_a3.cl b/OpenCL/m04500_a3.cl
index 2d5ff62..f2c59f2 100644 (file)
--- a/OpenCL/m04500_a3.cl
+++ b/OpenCL/m04500_a3.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04700_a0.cl b/OpenCL/m04700_a0.cl
index bc73392..3d06321 100644 (file)
--- a/OpenCL/m04700_a0.cl
+++ b/OpenCL/m04700_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04700_a1.cl b/OpenCL/m04700_a1.cl
index b71ef75..3f572d3 100644 (file)
--- a/OpenCL/m04700_a1.cl
+++ b/OpenCL/m04700_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04700_a3.cl b/OpenCL/m04700_a3.cl
index a1e13ee..a5f28cb 100644 (file)
--- a/OpenCL/m04700_a3.cl
+++ b/OpenCL/m04700_a3.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04800_a0.cl b/OpenCL/m04800_a0.cl
index 5df1807..72183d0 100644 (file)
--- a/OpenCL/m04800_a0.cl
+++ b/OpenCL/m04800_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m04800_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m04800_a1.cl b/OpenCL/m04800_a1.cl
index 7af41b7..4258491 100644 (file)
--- a/OpenCL/m04800_a1.cl
+++ b/OpenCL/m04800_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m04800_a3.cl b/OpenCL/m04800_a3.cl
index eea003d..52445bc 100644 (file)
--- a/OpenCL/m04800_a3.cl
+++ b/OpenCL/m04800_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m04800m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m04900_a0.cl b/OpenCL/m04900_a0.cl
index 15d8ec2..6ad2453 100644 (file)
--- a/OpenCL/m04900_a0.cl
+++ b/OpenCL/m04900_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m04900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m04900_a1.cl b/OpenCL/m04900_a1.cl
index 31b9ae4..79be593 100644 (file)
--- a/OpenCL/m04900_a1.cl
+++ b/OpenCL/m04900_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m04900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m04900_a3.cl b/OpenCL/m04900_a3.cl
index 9cf2ba0..b7b75b8 100644 (file)
--- a/OpenCL/m04900_a3.cl
+++ b/OpenCL/m04900_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m04900m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m05000_a0.cl b/OpenCL/m05000_a0.cl
index 7bada0e..8d53020 100644 (file)
--- a/OpenCL/m05000_a0.cl
+++ b/OpenCL/m05000_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 2
 #define DGST_R1 3
 #define DGST_R2 4
 #define DGST_R3 5
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u64 keccakf_rndc[24] =
 {

diff --git a/OpenCL/m05000_a1.cl b/OpenCL/m05000_a1.cl
index 7d22c13..b6ef979 100644 (file)
--- a/OpenCL/m05000_a1.cl
+++ b/OpenCL/m05000_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 2
 #define DGST_R1 3
 #define DGST_R2 4
 #define DGST_R3 5
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 keccakf_rndc[24] =
 {

diff --git a/OpenCL/m05000_a3.cl b/OpenCL/m05000_a3.cl
index e057e16..30c4ea1 100644 (file)
--- a/OpenCL/m05000_a3.cl
+++ b/OpenCL/m05000_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 2
 #define DGST_R1 3
 #define DGST_R2 4
 #define DGST_R3 5
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 keccakf_rndc[24] =
 {

diff --git a/OpenCL/m05100_a0.cl b/OpenCL/m05100_a0.cl
index 5f6593b..e3625a0 100644 (file)
--- a/OpenCL/m05100_a0.cl
+++ b/OpenCL/m05100_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m05100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m05100_a1.cl b/OpenCL/m05100_a1.cl
index 795f883..7b0f470 100644 (file)
--- a/OpenCL/m05100_a1.cl
+++ b/OpenCL/m05100_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m05100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m05100_a3.cl b/OpenCL/m05100_a3.cl
index 5a41ae2..7aa9226 100644 (file)
--- a/OpenCL/m05100_a3.cl
+++ b/OpenCL/m05100_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m05100m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m05200.cl b/OpenCL/m05200.cl
index abd2912..2fb4547 100644 (file)
--- a/OpenCL/m05200.cl
+++ b/OpenCL/m05200.cl
@@ -5,20 +5,20 @@
 
 #define _SHA256_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m05300_a0.cl b/OpenCL/m05300_a0.cl
index 943589d..0fc2f1a 100644 (file)
--- a/OpenCL/m05300_a0.cl
+++ b/OpenCL/m05300_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m05300_a1.cl b/OpenCL/m05300_a1.cl
index 9f4e6e9..7c53c3b 100644 (file)
--- a/OpenCL/m05300_a1.cl
+++ b/OpenCL/m05300_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m05300_a3.cl b/OpenCL/m05300_a3.cl
index ee2d43c..276c2a3 100644 (file)
--- a/OpenCL/m05300_a3.cl
+++ b/OpenCL/m05300_a3.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m05400_a0.cl b/OpenCL/m05400_a0.cl
index d881a58..a722c53 100644 (file)
--- a/OpenCL/m05400_a0.cl
+++ b/OpenCL/m05400_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m05400_a1.cl b/OpenCL/m05400_a1.cl
index 4958b39..ec2955d 100644 (file)
--- a/OpenCL/m05400_a1.cl
+++ b/OpenCL/m05400_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m05400_a3.cl b/OpenCL/m05400_a3.cl
index d4d91fb..f76af43 100644 (file)
--- a/OpenCL/m05400_a3.cl
+++ b/OpenCL/m05400_a3.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m05500_a0.cl b/OpenCL/m05500_a0.cl
index 82ad293..90b7ad6 100644 (file)
--- a/OpenCL/m05500_a0.cl
+++ b/OpenCL/m05500_a0.cl
@@ -10,20 +10,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m05500_a1.cl b/OpenCL/m05500_a1.cl
index be2994c..12d2d48 100644 (file)
--- a/OpenCL/m05500_a1.cl
+++ b/OpenCL/m05500_a1.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m05500_a3.cl b/OpenCL/m05500_a3.cl
index a9fc359..d77bb3b 100644 (file)
--- a/OpenCL/m05500_a3.cl
+++ b/OpenCL/m05500_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m05600_a0.cl b/OpenCL/m05600_a0.cl
index 74920b7..678cca8 100644 (file)
--- a/OpenCL/m05600_a0.cl
+++ b/OpenCL/m05600_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void md4_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m05600_a1.cl b/OpenCL/m05600_a1.cl
index e0d566b..8ce3e69 100644 (file)
--- a/OpenCL/m05600_a1.cl
+++ b/OpenCL/m05600_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md4_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m05600_a3.cl b/OpenCL/m05600_a3.cl
index 799dce0..90b28a5 100644 (file)
--- a/OpenCL/m05600_a3.cl
+++ b/OpenCL/m05600_a3.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md4_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m05800.cl b/OpenCL/m05800.cl
index 1c363b7..fda4748 100644 (file)
--- a/OpenCL/m05800.cl
+++ b/OpenCL/m05800.cl
@@ -5,20 +5,20 @@
 
 #define _SHA1_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m06000_a0.cl b/OpenCL/m06000_a0.cl
index 5f630f8..a9b4310 100644 (file)
--- a/OpenCL/m06000_a0.cl
+++ b/OpenCL/m06000_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void ripemd160_transform (const u32x w[16], u32x dgst[5])
 {

diff --git a/OpenCL/m06000_a1.cl b/OpenCL/m06000_a1.cl
index 29e546c..77d832e 100644 (file)
--- a/OpenCL/m06000_a1.cl
+++ b/OpenCL/m06000_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void ripemd160_transform (const u32x w[16], u32x dgst[5])
 {

diff --git a/OpenCL/m06000_a3.cl b/OpenCL/m06000_a3.cl
index ac3eab8..c9222cd 100644 (file)
--- a/OpenCL/m06000_a3.cl
+++ b/OpenCL/m06000_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void ripemd160_transform (const u32x w[16], u32x dgst[5])
 {

diff --git a/OpenCL/m06100_a0.cl b/OpenCL/m06100_a0.cl
index 3b0d23c..085f790 100644 (file)
--- a/OpenCL/m06100_a0.cl
+++ b/OpenCL/m06100_a0.cl
@@ -10,20 +10,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define R 10
 

diff --git a/OpenCL/m06100_a1.cl b/OpenCL/m06100_a1.cl
index 35005ae..2280f21 100644 (file)
--- a/OpenCL/m06100_a1.cl
+++ b/OpenCL/m06100_a1.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define R 10
 

diff --git a/OpenCL/m06100_a3.cl b/OpenCL/m06100_a3.cl
index d613d2e..c26d3fb 100644 (file)
--- a/OpenCL/m06100_a3.cl
+++ b/OpenCL/m06100_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define R 10
 

diff --git a/OpenCL/m06211.cl b/OpenCL/m06211.cl
index da07d43..f3895da 100644 (file)
--- a/OpenCL/m06211.cl
+++ b/OpenCL/m06211.cl
@@ -5,21 +5,21 @@
 
 #define _RIPEMD160_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 void ripemd160_transform (const u32 w[16], u32 dgst[5])
 {

diff --git a/OpenCL/m06212.cl b/OpenCL/m06212.cl
index 7262ac3..d06cfd8 100644 (file)
--- a/OpenCL/m06212.cl
+++ b/OpenCL/m06212.cl
@@ -5,21 +5,21 @@
 
 #define _RIPEMD160_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 void ripemd160_transform (const u32 w[16], u32 dgst[5])
 {

diff --git a/OpenCL/m06213.cl b/OpenCL/m06213.cl
index 0b49be0..61a85c1 100644 (file)
--- a/OpenCL/m06213.cl
+++ b/OpenCL/m06213.cl
@@ -5,21 +5,21 @@
 
 #define _RIPEMD160_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 void ripemd160_transform (const u32 w[16], u32 dgst[5])
 {

diff --git a/OpenCL/m06221.cl b/OpenCL/m06221.cl
index 5959263..be8041f 100644 (file)
--- a/OpenCL/m06221.cl
+++ b/OpenCL/m06221.cl
@@ -5,21 +5,21 @@
 
 #define _SHA512_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m06222.cl b/OpenCL/m06222.cl
index e416c66..1c09496 100644 (file)
--- a/OpenCL/m06222.cl
+++ b/OpenCL/m06222.cl
@@ -5,21 +5,21 @@
 
 #define _SHA512_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m06223.cl b/OpenCL/m06223.cl
index c2007b0..3e7aa8d 100644 (file)
--- a/OpenCL/m06223.cl
+++ b/OpenCL/m06223.cl
@@ -5,21 +5,21 @@
 
 #define _SHA512_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m06231.cl b/OpenCL/m06231.cl
index 90f248c..2ead1ea 100644 (file)
--- a/OpenCL/m06231.cl
+++ b/OpenCL/m06231.cl
@@ -7,21 +7,21 @@
 
 #define _WHIRLPOOL_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 #define R 10
 

diff --git a/OpenCL/m06232.cl b/OpenCL/m06232.cl
index 942de68..b10280e 100644 (file)
--- a/OpenCL/m06232.cl
+++ b/OpenCL/m06232.cl
@@ -7,21 +7,21 @@
 
 #define _WHIRLPOOL_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 #define R 10
 

diff --git a/OpenCL/m06233.cl b/OpenCL/m06233.cl
index acb6719..0235544 100644 (file)
--- a/OpenCL/m06233.cl
+++ b/OpenCL/m06233.cl
@@ -7,21 +7,21 @@
 
 #define _WHIRLPOOL_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 #define R 10
 

diff --git a/OpenCL/m06300.cl b/OpenCL/m06300.cl
index f4dcf90..cfdba3b 100644 (file)
--- a/OpenCL/m06300.cl
+++ b/OpenCL/m06300.cl
@@ -5,20 +5,20 @@
 
 #define _MD5_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 void md5_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4])
 {

diff --git a/OpenCL/m06400.cl b/OpenCL/m06400.cl
index 99e0dd8..8c632a5 100644 (file)
--- a/OpenCL/m06400.cl
+++ b/OpenCL/m06400.cl
@@ -5,20 +5,20 @@
 
 #define _SHA256_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m06500.cl b/OpenCL/m06500.cl
index b5d77e1..5f23af1 100644 (file)
--- a/OpenCL/m06500.cl
+++ b/OpenCL/m06500.cl
@@ -5,20 +5,20 @@
 
 #define _SHA512_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m06600.cl b/OpenCL/m06600.cl
index 6ee0523..ca2f14e 100644 (file)
--- a/OpenCL/m06600.cl
+++ b/OpenCL/m06600.cl
@@ -7,20 +7,20 @@
 
 #define _SHA1_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m06700.cl b/OpenCL/m06700.cl
index 3f8aceb..f8b5ff8 100644 (file)
--- a/OpenCL/m06700.cl
+++ b/OpenCL/m06700.cl
@@ -5,20 +5,20 @@
 
 #define _SHA1_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 void sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5])
 {

diff --git a/OpenCL/m06800.cl b/OpenCL/m06800.cl
index d2e6915..6744d6a 100644 (file)
--- a/OpenCL/m06800.cl
+++ b/OpenCL/m06800.cl
@@ -7,20 +7,20 @@
 
 #define _SHA256_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m06900_a0.cl b/OpenCL/m06900_a0.cl
index 82833bf..bbefaff 100644 (file)
--- a/OpenCL/m06900_a0.cl
+++ b/OpenCL/m06900_a0.cl
@@ -10,20 +10,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 c_tables[4][256] =
 {

diff --git a/OpenCL/m06900_a1.cl b/OpenCL/m06900_a1.cl
index f7a0765..fedc7fa 100644 (file)
--- a/OpenCL/m06900_a1.cl
+++ b/OpenCL/m06900_a1.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 c_tables[4][256] =
 {

diff --git a/OpenCL/m06900_a3.cl b/OpenCL/m06900_a3.cl
index dc7c705..66f6f82 100644 (file)
--- a/OpenCL/m06900_a3.cl
+++ b/OpenCL/m06900_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 c_tables[4][256] =
 {

diff --git a/OpenCL/m07100.cl b/OpenCL/m07100.cl
index 1147122..6f64405 100644 (file)
--- a/OpenCL/m07100.cl
+++ b/OpenCL/m07100.cl
@@ -7,21 +7,21 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m07300_a0.cl b/OpenCL/m07300_a0.cl
index 9d4dccf..27c7a4f 100644 (file)
--- a/OpenCL/m07300_a0.cl
+++ b/OpenCL/m07300_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m07300_a1.cl b/OpenCL/m07300_a1.cl
index 2cfa5ca..b78560c 100644 (file)
--- a/OpenCL/m07300_a1.cl
+++ b/OpenCL/m07300_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m07300_a3.cl b/OpenCL/m07300_a3.cl
index 944ec2e..25a0495 100644 (file)
--- a/OpenCL/m07300_a3.cl
+++ b/OpenCL/m07300_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m07400.cl b/OpenCL/m07400.cl
index 0e22a5e..3fc4600 100644 (file)
--- a/OpenCL/m07400.cl
+++ b/OpenCL/m07400.cl
@@ -5,20 +5,20 @@
 
 #define _SHA256_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m07500_a0.cl b/OpenCL/m07500_a0.cl
index 0393d8d..ea2e219 100644 (file)
--- a/OpenCL/m07500_a0.cl
+++ b/OpenCL/m07500_a0.cl
@@ -8,20 +8,20 @@
 //shared mem too small
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m07500_a1.cl b/OpenCL/m07500_a1.cl
index 6983b16..673df0a 100644 (file)
--- a/OpenCL/m07500_a1.cl
+++ b/OpenCL/m07500_a1.cl
@@ -8,18 +8,18 @@
 //shared mem too small
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m07500_a3.cl b/OpenCL/m07500_a3.cl
index 1e8eb93..dd57cce 100644 (file)
--- a/OpenCL/m07500_a3.cl
+++ b/OpenCL/m07500_a3.cl
@@ -10,18 +10,18 @@
 //shared mem too small
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m07600_a0.cl b/OpenCL/m07600_a0.cl
index 86a0012..15d1ed1 100644 (file)
--- a/OpenCL/m07600_a0.cl
+++ b/OpenCL/m07600_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m07600_a1.cl b/OpenCL/m07600_a1.cl
index 531d5f3..b904ed4 100644 (file)
--- a/OpenCL/m07600_a1.cl
+++ b/OpenCL/m07600_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m07600_a3.cl b/OpenCL/m07600_a3.cl
index 2e068e7..717d7de 100644 (file)
--- a/OpenCL/m07600_a3.cl
+++ b/OpenCL/m07600_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m07700_a0.cl b/OpenCL/m07700_a0.cl
index 2755132..671d02d 100644 (file)
--- a/OpenCL/m07700_a0.cl
+++ b/OpenCL/m07700_a0.cl
@@ -8,20 +8,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define GETCHAR(a,p)  (((a)[(p) / 4] >> (((p) & 3) * 8)) & 0xff)
 #define PUTCHAR(a,p,c) ((a)[(p) / 4] = (((a)[(p) / 4] & ~(0xff << (((p) & 3) * 8))) | ((c) << (((p) & 3) * 8))))

diff --git a/OpenCL/m07700_a1.cl b/OpenCL/m07700_a1.cl
index 8eb9054..bf98509 100644 (file)
--- a/OpenCL/m07700_a1.cl
+++ b/OpenCL/m07700_a1.cl
@@ -8,18 +8,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define GETCHAR(a,p)  (((a)[(p) / 4] >> (((p) & 3) * 8)) & 0xff)
 #define PUTCHAR(a,p,c) ((a)[(p) / 4] = (((a)[(p) / 4] & ~(0xff << (((p) & 3) * 8))) | ((c) << (((p) & 3) * 8))))

diff --git a/OpenCL/m07700_a3.cl b/OpenCL/m07700_a3.cl
index dac0379..293cc49 100644 (file)
--- a/OpenCL/m07700_a3.cl
+++ b/OpenCL/m07700_a3.cl
@@ -8,18 +8,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define GETCHAR(a,p)  (((a)[(p) / 4] >> (((p) & 3) * 8)) & 0xff)
 #define PUTCHAR(a,p,c) ((a)[(p) / 4] = (((a)[(p) / 4] & ~(0xff << (((p) & 3) * 8))) | ((c) << (((p) & 3) * 8))))

diff --git a/OpenCL/m07800_a0.cl b/OpenCL/m07800_a0.cl
index b4b98db..308b582 100644 (file)
--- a/OpenCL/m07800_a0.cl
+++ b/OpenCL/m07800_a0.cl
@@ -8,20 +8,20 @@
 //incompatible data-dependant code
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define GETSHIFTEDINT(a,n) amd_bytealign ((a)[((n)/4)+1], (a)[((n)/4)+0], (n))
 

diff --git a/OpenCL/m07800_a1.cl b/OpenCL/m07800_a1.cl
index feb36f1..4fd07eb 100644 (file)
--- a/OpenCL/m07800_a1.cl
+++ b/OpenCL/m07800_a1.cl
@@ -8,18 +8,18 @@
 //incompatible data-dependant code
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define GETSHIFTEDINT(a,n) amd_bytealign ((a)[((n)/4)+1], (a)[((n)/4)+0], (n))
 

diff --git a/OpenCL/m07800_a3.cl b/OpenCL/m07800_a3.cl
index 3b75ace..7d1ce5a 100644 (file)
--- a/OpenCL/m07800_a3.cl
+++ b/OpenCL/m07800_a3.cl
@@ -8,18 +8,18 @@
 //incompatible data-dependant code
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define GETSHIFTEDINT(a,n) amd_bytealign ((a)[((n)/4)+1], (a)[((n)/4)+0], (n))
 

diff --git a/OpenCL/m07900.cl b/OpenCL/m07900.cl
index 7226bec..49389d8 100644 (file)
--- a/OpenCL/m07900.cl
+++ b/OpenCL/m07900.cl
@@ -5,20 +5,20 @@
 
 #define _SHA512_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m08000_a0.cl b/OpenCL/m08000_a0.cl
index 33266e0..046b84e 100644 (file)
--- a/OpenCL/m08000_a0.cl
+++ b/OpenCL/m08000_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m08000_a1.cl b/OpenCL/m08000_a1.cl
index efbc2bb..9ac3580 100644 (file)
--- a/OpenCL/m08000_a1.cl
+++ b/OpenCL/m08000_a1.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m08000_a3.cl b/OpenCL/m08000_a3.cl
index b670305..01f66fb 100644 (file)
--- a/OpenCL/m08000_a3.cl
+++ b/OpenCL/m08000_a3.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m08100_a0.cl b/OpenCL/m08100_a0.cl
index 95b76e2..32c8a61 100644 (file)
--- a/OpenCL/m08100_a0.cl
+++ b/OpenCL/m08100_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m08100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m08100_a1.cl b/OpenCL/m08100_a1.cl
index 25980ec..a7c2d8d 100644 (file)
--- a/OpenCL/m08100_a1.cl
+++ b/OpenCL/m08100_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m08100_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m08100_a3.cl b/OpenCL/m08100_a3.cl
index 4848662..58240d1 100644 (file)
--- a/OpenCL/m08100_a3.cl
+++ b/OpenCL/m08100_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m08100m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m08200.cl b/OpenCL/m08200.cl
index dc7b768..ae7e6de 100644 (file)
--- a/OpenCL/m08200.cl
+++ b/OpenCL/m08200.cl
@@ -5,20 +5,20 @@
 
 #define _CLOUDKEY_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m08300_a0.cl b/OpenCL/m08300_a0.cl
index 904f680..84dfa7f 100644 (file)
--- a/OpenCL/m08300_a0.cl
+++ b/OpenCL/m08300_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m08300_a1.cl b/OpenCL/m08300_a1.cl
index e36b357..b22986f 100644 (file)
--- a/OpenCL/m08300_a1.cl
+++ b/OpenCL/m08300_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m08300_a3.cl b/OpenCL/m08300_a3.cl
index 9de6a42..b0e73e1 100644 (file)
--- a/OpenCL/m08300_a3.cl
+++ b/OpenCL/m08300_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m08400_a0.cl b/OpenCL/m08400_a0.cl
index 2fa98bd..06df81f 100644 (file)
--- a/OpenCL/m08400_a0.cl
+++ b/OpenCL/m08400_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m08400_a1.cl b/OpenCL/m08400_a1.cl
index 5cb4be4..e408c05 100644 (file)
--- a/OpenCL/m08400_a1.cl
+++ b/OpenCL/m08400_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m08400_a3.cl b/OpenCL/m08400_a3.cl
index 7a089ad..ce4c926 100644 (file)
--- a/OpenCL/m08400_a3.cl
+++ b/OpenCL/m08400_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8_le(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m08500_a0.cl b/OpenCL/m08500_a0.cl
index d21d101..6874a7e 100644 (file)
--- a/OpenCL/m08500_a0.cl
+++ b/OpenCL/m08500_a0.cl
@@ -10,20 +10,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m08500_a1.cl b/OpenCL/m08500_a1.cl
index 20f2599..d007a1c 100644 (file)
--- a/OpenCL/m08500_a1.cl
+++ b/OpenCL/m08500_a1.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m08500_a3.cl b/OpenCL/m08500_a3.cl
index a97d62d..6e5058a 100644 (file)
--- a/OpenCL/m08500_a3.cl
+++ b/OpenCL/m08500_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m08600_a0.cl b/OpenCL/m08600_a0.cl
index c4a043c..c6e58ed 100644 (file)
--- a/OpenCL/m08600_a0.cl
+++ b/OpenCL/m08600_a0.cl
@@ -11,20 +11,20 @@
 //incompatible
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 lotus_magic_table[256] =
 {

diff --git a/OpenCL/m08600_a1.cl b/OpenCL/m08600_a1.cl
index 58be048..517bbe1 100644 (file)
--- a/OpenCL/m08600_a1.cl
+++ b/OpenCL/m08600_a1.cl
@@ -10,18 +10,18 @@
 //incompatible
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 lotus_magic_table[256] =
 {

diff --git a/OpenCL/m08600_a3.cl b/OpenCL/m08600_a3.cl
index 5b7d924..5fe5e44 100644 (file)
--- a/OpenCL/m08600_a3.cl
+++ b/OpenCL/m08600_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 lotus_magic_table[256] =
 {

diff --git a/OpenCL/m08700_a0.cl b/OpenCL/m08700_a0.cl
index 1bcb8d6..9506789 100644 (file)
--- a/OpenCL/m08700_a0.cl
+++ b/OpenCL/m08700_a0.cl
@@ -11,20 +11,20 @@
 //incompatible
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 lotus_magic_table[256] =
 {

diff --git a/OpenCL/m08700_a1.cl b/OpenCL/m08700_a1.cl
index ed6d197..00650f1 100644 (file)
--- a/OpenCL/m08700_a1.cl
+++ b/OpenCL/m08700_a1.cl
@@ -10,18 +10,18 @@
 //incompatible
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 lotus_magic_table[256] =
 {

diff --git a/OpenCL/m08700_a3.cl b/OpenCL/m08700_a3.cl
index b6fd2d3..f3bd318 100644 (file)
--- a/OpenCL/m08700_a3.cl
+++ b/OpenCL/m08700_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 lotus_magic_table[256] =
 {

diff --git a/OpenCL/m08800.cl b/OpenCL/m08800.cl
index 38a3033..50c5ca1 100644 (file)
--- a/OpenCL/m08800.cl
+++ b/OpenCL/m08800.cl
@@ -7,17 +7,17 @@
 
 #define _ANDROIDFDE_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m08900.cl b/OpenCL/m08900.cl
index 85ad68d..c2a718a 100644 (file)
--- a/OpenCL/m08900.cl
+++ b/OpenCL/m08900.cl
@@ -5,20 +5,20 @@
 
 #define _SCRYPT_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m09000.cl b/OpenCL/m09000.cl
index 210b620..7b1166b 100644 (file)
--- a/OpenCL/m09000.cl
+++ b/OpenCL/m09000.cl
@@ -5,20 +5,20 @@
 
 #define _PSAFE2_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 // http://www.schneier.com/code/constants.txt
 

diff --git a/OpenCL/m09100.cl b/OpenCL/m09100.cl
index 4efe528..2d694d6 100644 (file)
--- a/OpenCL/m09100.cl
+++ b/OpenCL/m09100.cl
@@ -7,23 +7,23 @@
 
 #define _LOTUS8_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
+#include "inc_hash_functions.cl"
 
 #undef _SHA1_
 
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant char lotus64_table[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+/";
 

diff --git a/OpenCL/m09400.cl b/OpenCL/m09400.cl
index d9249dc..d28c6b2 100644 (file)
--- a/OpenCL/m09400.cl
+++ b/OpenCL/m09400.cl
@@ -7,20 +7,20 @@
 
 #define _OFFICE2007_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m09500.cl b/OpenCL/m09500.cl
index 3445f43..a51d1fc 100644 (file)
--- a/OpenCL/m09500.cl
+++ b/OpenCL/m09500.cl
@@ -7,20 +7,20 @@
 
 #define _OFFICE2010_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m09600.cl b/OpenCL/m09600.cl
index eb28f6c..18acd46 100644 (file)
--- a/OpenCL/m09600.cl
+++ b/OpenCL/m09600.cl
@@ -7,20 +7,20 @@
 
 #define _OFFICE2013_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m09700_a0.cl b/OpenCL/m09700_a0.cl
index 4fa60b9..0c71129 100644 (file)
--- a/OpenCL/m09700_a0.cl
+++ b/OpenCL/m09700_a0.cl
@@ -8,20 +8,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09700_a1.cl b/OpenCL/m09700_a1.cl
index 65162f7..1fcb15e 100644 (file)
--- a/OpenCL/m09700_a1.cl
+++ b/OpenCL/m09700_a1.cl
@@ -8,18 +8,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09700_a3.cl b/OpenCL/m09700_a3.cl
index 45128af..d9a4cf9 100644 (file)
--- a/OpenCL/m09700_a3.cl
+++ b/OpenCL/m09700_a3.cl
@@ -7,18 +7,18 @@
 
 #define _OLDOFFICE01_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09710_a0.cl b/OpenCL/m09710_a0.cl
index b992e8d..040abd5 100644 (file)
--- a/OpenCL/m09710_a0.cl
+++ b/OpenCL/m09710_a0.cl
@@ -8,20 +8,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09710_a1.cl b/OpenCL/m09710_a1.cl
index c761e44..74e119e 100644 (file)
--- a/OpenCL/m09710_a1.cl
+++ b/OpenCL/m09710_a1.cl
@@ -8,18 +8,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09710_a3.cl b/OpenCL/m09710_a3.cl
index 61f23d1..17b4adf 100644 (file)
--- a/OpenCL/m09710_a3.cl
+++ b/OpenCL/m09710_a3.cl
@@ -10,18 +10,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09720_a0.cl b/OpenCL/m09720_a0.cl
index 6721d00..8aeb1e2 100644 (file)
--- a/OpenCL/m09720_a0.cl
+++ b/OpenCL/m09720_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m09720_a1.cl b/OpenCL/m09720_a1.cl
index 091d041..b0c6bba 100644 (file)
--- a/OpenCL/m09720_a1.cl
+++ b/OpenCL/m09720_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m09720_a3.cl b/OpenCL/m09720_a3.cl
index 0b904da..66b5ccd 100644 (file)
--- a/OpenCL/m09720_a3.cl
+++ b/OpenCL/m09720_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4])
 {

diff --git a/OpenCL/m09800_a0.cl b/OpenCL/m09800_a0.cl
index 31f6355..2267b1a 100644 (file)
--- a/OpenCL/m09800_a0.cl
+++ b/OpenCL/m09800_a0.cl
@@ -8,20 +8,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09800_a1.cl b/OpenCL/m09800_a1.cl
index 1c11f01..48cc909 100644 (file)
--- a/OpenCL/m09800_a1.cl
+++ b/OpenCL/m09800_a1.cl
@@ -8,18 +8,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09800_a3.cl b/OpenCL/m09800_a3.cl
index 6fc92c0..eaf1fa8 100644 (file)
--- a/OpenCL/m09800_a3.cl
+++ b/OpenCL/m09800_a3.cl
@@ -7,18 +7,18 @@
 
 #define _OLDOFFICE34_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09810_a0.cl b/OpenCL/m09810_a0.cl
index 40fb8a0..b27ee74 100644 (file)
--- a/OpenCL/m09810_a0.cl
+++ b/OpenCL/m09810_a0.cl
@@ -8,20 +8,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09810_a1.cl b/OpenCL/m09810_a1.cl
index 527b1bd..bf1ae52 100644 (file)
--- a/OpenCL/m09810_a1.cl
+++ b/OpenCL/m09810_a1.cl
@@ -8,18 +8,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09810_a3.cl b/OpenCL/m09810_a3.cl
index 5bfc896..56cf7c7 100644 (file)
--- a/OpenCL/m09810_a3.cl
+++ b/OpenCL/m09810_a3.cl
@@ -10,18 +10,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m09820_a0.cl b/OpenCL/m09820_a0.cl
index 8e780bc..89dda1a 100644 (file)
--- a/OpenCL/m09820_a0.cl
+++ b/OpenCL/m09820_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m09820_a1.cl b/OpenCL/m09820_a1.cl
index f8025e5..b8075b3 100644 (file)
--- a/OpenCL/m09820_a1.cl
+++ b/OpenCL/m09820_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m09820_a3.cl b/OpenCL/m09820_a3.cl
index 3f8b332..0b9fc57 100644 (file)
--- a/OpenCL/m09820_a3.cl
+++ b/OpenCL/m09820_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5])
 {

diff --git a/OpenCL/m09900_a0.cl b/OpenCL/m09900_a0.cl
index 91ded72..fddcef8 100644 (file)
--- a/OpenCL/m09900_a0.cl
+++ b/OpenCL/m09900_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m09900_m04 (__global pw_t *pws, __global kernel_rule_t *  rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m09900_a1.cl b/OpenCL/m09900_a1.cl
index e6f051d..0e63e55 100644 (file)
--- a/OpenCL/m09900_a1.cl
+++ b/OpenCL/m09900_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m09900_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m09900_a3.cl b/OpenCL/m09900_a3.cl
index a11bf4c..30bae9b 100644 (file)
--- a/OpenCL/m09900_a3.cl
+++ b/OpenCL/m09900_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m09900m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m10100_a0.cl b/OpenCL/m10100_a0.cl
index 269aff3..ad82fb4 100644 (file)
--- a/OpenCL/m10100_a0.cl
+++ b/OpenCL/m10100_a0.cl
@@ -8,20 +8,20 @@
 //incompatible to simd
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define SIPROUND(v0,v1,v2,v3) \
   (v0) += (v1);               \

diff --git a/OpenCL/m10100_a1.cl b/OpenCL/m10100_a1.cl
index 345ad16..b797008 100644 (file)
--- a/OpenCL/m10100_a1.cl
+++ b/OpenCL/m10100_a1.cl
@@ -8,18 +8,18 @@
 //incompatible to simd
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define SIPROUND(v0,v1,v2,v3) \
   (v0) += (v1);               \

diff --git a/OpenCL/m10100_a3.cl b/OpenCL/m10100_a3.cl
index 116a7cf..ded29d3 100644 (file)
--- a/OpenCL/m10100_a3.cl
+++ b/OpenCL/m10100_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define SIPROUND(v0,v1,v2,v3) \
   (v0) += (v1);               \

diff --git a/OpenCL/m10300.cl b/OpenCL/m10300.cl
index 2493658..2dc8a99 100644 (file)
--- a/OpenCL/m10300.cl
+++ b/OpenCL/m10300.cl
@@ -5,20 +5,20 @@
 
 #define _SHA1_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 void sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5])
 {

diff --git a/OpenCL/m10400_a0.cl b/OpenCL/m10400_a0.cl
index 90cd0bc..9094a96 100644 (file)
--- a/OpenCL/m10400_a0.cl
+++ b/OpenCL/m10400_a0.cl
@@ -10,20 +10,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10400_a1.cl b/OpenCL/m10400_a1.cl
index 093af29..df58488 100644 (file)
--- a/OpenCL/m10400_a1.cl
+++ b/OpenCL/m10400_a1.cl
@@ -10,18 +10,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10400_a3.cl b/OpenCL/m10400_a3.cl
index 92ebfe5..b26f455 100644 (file)
--- a/OpenCL/m10400_a3.cl
+++ b/OpenCL/m10400_a3.cl
@@ -10,18 +10,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10410_a0.cl b/OpenCL/m10410_a0.cl
index 518b84c..51b1d74 100644 (file)
--- a/OpenCL/m10410_a0.cl
+++ b/OpenCL/m10410_a0.cl
@@ -10,20 +10,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10410_a1.cl b/OpenCL/m10410_a1.cl
index ec226b8..c5933ef 100644 (file)
--- a/OpenCL/m10410_a1.cl
+++ b/OpenCL/m10410_a1.cl
@@ -10,18 +10,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10410_a3.cl b/OpenCL/m10410_a3.cl
index d43b657..b56c678 100644 (file)
--- a/OpenCL/m10410_a3.cl
+++ b/OpenCL/m10410_a3.cl
@@ -10,18 +10,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10420_a0.cl b/OpenCL/m10420_a0.cl
index 9b69a25..f9d295b 100644 (file)
--- a/OpenCL/m10420_a0.cl
+++ b/OpenCL/m10420_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10420_a1.cl b/OpenCL/m10420_a1.cl
index 42cf320..9126e2c 100644 (file)
--- a/OpenCL/m10420_a1.cl
+++ b/OpenCL/m10420_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10420_a3.cl b/OpenCL/m10420_a3.cl
index 93b84c4..a7abb23 100644 (file)
--- a/OpenCL/m10420_a3.cl
+++ b/OpenCL/m10420_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10500.cl b/OpenCL/m10500.cl
index a24469f..721bfc5 100644 (file)
--- a/OpenCL/m10500.cl
+++ b/OpenCL/m10500.cl
@@ -5,20 +5,20 @@
 
 #define _MD5_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 padding[8] =
 {

diff --git a/OpenCL/m10700.cl b/OpenCL/m10700.cl
index d606f2d..e0a7389 100644 (file)
--- a/OpenCL/m10700.cl
+++ b/OpenCL/m10700.cl
@@ -7,20 +7,20 @@
 
 #define _PDF17L8_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m10800_a0.cl b/OpenCL/m10800_a0.cl
index 17f7540..c7de147 100644 (file)
--- a/OpenCL/m10800_a0.cl
+++ b/OpenCL/m10800_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 6
 #define DGST_R1 7
 #define DGST_R2 4
 #define DGST_R3 5
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha384[80] =
 {

diff --git a/OpenCL/m10800_a1.cl b/OpenCL/m10800_a1.cl
index bcb0c58..a3897b7 100644 (file)
--- a/OpenCL/m10800_a1.cl
+++ b/OpenCL/m10800_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 6
 #define DGST_R1 7
 #define DGST_R2 4
 #define DGST_R3 5
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha384[80] =
 {

diff --git a/OpenCL/m10800_a3.cl b/OpenCL/m10800_a3.cl
index 7bfc741..fae6a72 100644 (file)
--- a/OpenCL/m10800_a3.cl
+++ b/OpenCL/m10800_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 6
 #define DGST_R1 7
 #define DGST_R2 4
 #define DGST_R3 5
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u64 k_sha384[80] =
 {

diff --git a/OpenCL/m10900.cl b/OpenCL/m10900.cl
index 9427fab..275b049 100644 (file)
--- a/OpenCL/m10900.cl
+++ b/OpenCL/m10900.cl
@@ -7,21 +7,21 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m11000_a0.cl b/OpenCL/m11000_a0.cl
index a209c8a..2b5970d 100644 (file)
--- a/OpenCL/m11000_a0.cl
+++ b/OpenCL/m11000_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m11000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m11000_a1.cl b/OpenCL/m11000_a1.cl
index f6c8e7a..808914e 100644 (file)
--- a/OpenCL/m11000_a1.cl
+++ b/OpenCL/m11000_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m11000_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m11000_a3.cl b/OpenCL/m11000_a3.cl
index 0c42ddf..569d149 100644 (file)
--- a/OpenCL/m11000_a3.cl
+++ b/OpenCL/m11000_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m11000m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m11100_a0.cl b/OpenCL/m11100_a0.cl
index 8ab430a..ec1f0fc 100644 (file)
--- a/OpenCL/m11100_a0.cl
+++ b/OpenCL/m11100_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m11100_a1.cl b/OpenCL/m11100_a1.cl
index 7ec184a..50ec905 100644 (file)
--- a/OpenCL/m11100_a1.cl
+++ b/OpenCL/m11100_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m11100_a3.cl b/OpenCL/m11100_a3.cl
index 2d00c49..a5d860c 100644 (file)
--- a/OpenCL/m11100_a3.cl
+++ b/OpenCL/m11100_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m11200_a0.cl b/OpenCL/m11200_a0.cl
index 1e39af0..f63ec3b 100644 (file)
--- a/OpenCL/m11200_a0.cl
+++ b/OpenCL/m11200_a0.cl
@@ -7,20 +7,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m11200_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m11200_a1.cl b/OpenCL/m11200_a1.cl
index 062ec3b..0790a46 100644 (file)
--- a/OpenCL/m11200_a1.cl
+++ b/OpenCL/m11200_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m11200_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m11200_a3.cl b/OpenCL/m11200_a3.cl
index eec3317..4d1ecfb 100644 (file)
--- a/OpenCL/m11200_a3.cl
+++ b/OpenCL/m11200_a3.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m11200m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m11300.cl b/OpenCL/m11300.cl
index 3bfa242..df9fdd5 100644 (file)
--- a/OpenCL/m11300.cl
+++ b/OpenCL/m11300.cl
@@ -7,17 +7,17 @@
 
 #define _SHA512_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m11400_a0.cl b/OpenCL/m11400_a0.cl
index 5ac6101..2d3981b 100644 (file)
--- a/OpenCL/m11400_a0.cl
+++ b/OpenCL/m11400_a0.cl
@@ -10,20 +10,20 @@
 //incompatible because of brances
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m11400_a1.cl b/OpenCL/m11400_a1.cl
index d16b4ee..c15333d 100644 (file)
--- a/OpenCL/m11400_a1.cl
+++ b/OpenCL/m11400_a1.cl
@@ -8,18 +8,18 @@
 //incompatible because of brances
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m11400_a3.cl b/OpenCL/m11400_a3.cl
index 8086990..4caa348 100644 (file)
--- a/OpenCL/m11400_a3.cl
+++ b/OpenCL/m11400_a3.cl
@@ -11,18 +11,18 @@
 //incompatible because of brances
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 3
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m11500_a0.cl b/OpenCL/m11500_a0.cl
index 3bcc15b..088881e 100644 (file)
--- a/OpenCL/m11500_a0.cl
+++ b/OpenCL/m11500_a0.cl
@@ -10,20 +10,20 @@
 //incompatible because of branches
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __constant u32 crc32tab[0x100] =
 {

diff --git a/OpenCL/m11500_a1.cl b/OpenCL/m11500_a1.cl
index 3dd6c46..6d3bb7f 100644 (file)
--- a/OpenCL/m11500_a1.cl
+++ b/OpenCL/m11500_a1.cl
@@ -8,18 +8,18 @@
 //incompatible because of branches
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 crc32tab[0x100] =
 {

diff --git a/OpenCL/m11500_a3.cl b/OpenCL/m11500_a3.cl
index c8e9986..4af75ed 100644 (file)
--- a/OpenCL/m11500_a3.cl
+++ b/OpenCL/m11500_a3.cl
@@ -10,18 +10,18 @@
 //incompatible because of branches
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __constant u32 crc32tab[0x100] =
 {

diff --git a/OpenCL/m11600.cl b/OpenCL/m11600.cl
index c7af848..00ae013 100644 (file)
--- a/OpenCL/m11600.cl
+++ b/OpenCL/m11600.cl
@@ -7,20 +7,20 @@
 
 #define _SEVEN_ZIP_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m11700_a0.cl b/OpenCL/m11700_a0.cl
index 2e93185..db98b14 100644 (file)
--- a/OpenCL/m11700_a0.cl
+++ b/OpenCL/m11700_a0.cl
@@ -11,20 +11,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define INITVAL 0x0101010101010101
 

diff --git a/OpenCL/m11700_a1.cl b/OpenCL/m11700_a1.cl
index 36ce0b9..e0320dd 100644 (file)
--- a/OpenCL/m11700_a1.cl
+++ b/OpenCL/m11700_a1.cl
@@ -10,18 +10,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define INITVAL 0x0101010101010101
 

diff --git a/OpenCL/m11700_a3.cl b/OpenCL/m11700_a3.cl
index 1569b1f..0224f23 100644 (file)
--- a/OpenCL/m11700_a3.cl
+++ b/OpenCL/m11700_a3.cl
@@ -11,18 +11,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define INITVAL 0x0101010101010101
 

diff --git a/OpenCL/m11800_a0.cl b/OpenCL/m11800_a0.cl
index f282c3e..87e63b5 100644 (file)
--- a/OpenCL/m11800_a0.cl
+++ b/OpenCL/m11800_a0.cl
@@ -11,20 +11,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #define INITVAL 0
 

diff --git a/OpenCL/m11800_a1.cl b/OpenCL/m11800_a1.cl
index cf8392c..0cd6cf8 100644 (file)
--- a/OpenCL/m11800_a1.cl
+++ b/OpenCL/m11800_a1.cl
@@ -10,18 +10,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define INITVAL 0
 

diff --git a/OpenCL/m11800_a3.cl b/OpenCL/m11800_a3.cl
index 2370800..4ef9bb0 100644 (file)
--- a/OpenCL/m11800_a3.cl
+++ b/OpenCL/m11800_a3.cl
@@ -11,18 +11,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #define INITVAL 0
 

diff --git a/OpenCL/m11900.cl b/OpenCL/m11900.cl
index 53eec8f..ffa0f6c 100644 (file)
--- a/OpenCL/m11900.cl
+++ b/OpenCL/m11900.cl
@@ -7,21 +7,21 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 void md5_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4])
 {

diff --git a/OpenCL/m12000.cl b/OpenCL/m12000.cl
index ffbcf86..0619bcd 100644 (file)
--- a/OpenCL/m12000.cl
+++ b/OpenCL/m12000.cl
@@ -7,21 +7,21 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 void sha1_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5])
 {

diff --git a/OpenCL/m12200.cl b/OpenCL/m12200.cl
index 7b789bc..13af738 100644 (file)
--- a/OpenCL/m12200.cl
+++ b/OpenCL/m12200.cl
@@ -5,20 +5,20 @@
 
 #define _SHA512_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m12300.cl b/OpenCL/m12300.cl
index 5430b4f..3a87faa 100644 (file)
--- a/OpenCL/m12300.cl
+++ b/OpenCL/m12300.cl
@@ -5,20 +5,20 @@
 
 #define _PBKDF2_SHA512_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u64 k_sha512[80] =
 {

diff --git a/OpenCL/m12400.cl b/OpenCL/m12400.cl
index 7902c5d..d32f9b0 100644 (file)
--- a/OpenCL/m12400.cl
+++ b/OpenCL/m12400.cl
@@ -7,20 +7,20 @@
 
 #define _DES_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 #define PERM_OP(a,b,tt,n,m) \
 {                           \

diff --git a/OpenCL/m12500.cl b/OpenCL/m12500.cl
index 3292bc1..8c6b6e3 100644 (file)
--- a/OpenCL/m12500.cl
+++ b/OpenCL/m12500.cl
@@ -7,21 +7,21 @@
 
 #define _RAR3_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
+#include "inc_hash_functions.cl"
 
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 #define ROUNDS 0x40000
 

diff --git a/OpenCL/m12600_a0.cl b/OpenCL/m12600_a0.cl
index d9fe579..a6368f4 100644 (file)
--- a/OpenCL/m12600_a0.cl
+++ b/OpenCL/m12600_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m12600_a1.cl b/OpenCL/m12600_a1.cl
index 9b527c9..14f9018 100644 (file)
--- a/OpenCL/m12600_a1.cl
+++ b/OpenCL/m12600_a1.cl
@@ -7,18 +7,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m12600_a3.cl b/OpenCL/m12600_a3.cl
index dc8b835..dd6c899 100644 (file)
--- a/OpenCL/m12600_a3.cl
+++ b/OpenCL/m12600_a3.cl
@@ -10,18 +10,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 7
 #define DGST_R2 2
 #define DGST_R3 6
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 #if   VECT_SIZE == 1
 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)])

diff --git a/OpenCL/m12700.cl b/OpenCL/m12700.cl
index 1e51e35..4175979 100644 (file)
--- a/OpenCL/m12700.cl
+++ b/OpenCL/m12700.cl
@@ -7,20 +7,20 @@
 
 #define _SHA1_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m12800.cl b/OpenCL/m12800.cl
index a9e3f93..d138fc7 100644 (file)
--- a/OpenCL/m12800.cl
+++ b/OpenCL/m12800.cl
@@ -5,21 +5,21 @@
 
 #define _MS_DRSR_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
+#include "inc_hash_functions.cl"
 
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 #define uint_to_hex_lower8(i) l_bin2asc[(i)]
 

diff --git a/OpenCL/m12900.cl b/OpenCL/m12900.cl
index 817a819..589c170 100644 (file)
--- a/OpenCL/m12900.cl
+++ b/OpenCL/m12900.cl
@@ -5,21 +5,21 @@
 
 #define _ANDROIDFDE_SAMSUNG_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
+#include "inc_hash_functions.cl"
 
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m13000.cl b/OpenCL/m13000.cl
index 267081c..c07d8da 100644 (file)
--- a/OpenCL/m13000.cl
+++ b/OpenCL/m13000.cl
@@ -5,21 +5,21 @@
 
 #define _RAR5_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
+#include "inc_hash_functions.cl"
 
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m13100_a0.cl b/OpenCL/m13100_a0.cl
index bd16342..3544eaa 100644 (file)
--- a/OpenCL/m13100_a0.cl
+++ b/OpenCL/m13100_a0.cl
@@ -11,20 +11,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m13100_a1.cl b/OpenCL/m13100_a1.cl
index c49f2fd..2aba9d2 100644 (file)
--- a/OpenCL/m13100_a1.cl
+++ b/OpenCL/m13100_a1.cl
@@ -11,18 +11,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m13100_a3.cl b/OpenCL/m13100_a3.cl
index b549673..6113ae1 100644 (file)
--- a/OpenCL/m13100_a3.cl
+++ b/OpenCL/m13100_a3.cl
@@ -11,18 +11,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 typedef struct
 {

diff --git a/OpenCL/m13200.cl b/OpenCL/m13200.cl
index 6027cca..a104b16 100644 (file)
--- a/OpenCL/m13200.cl
+++ b/OpenCL/m13200.cl
@@ -7,17 +7,17 @@
 
 #define _SHA1_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m13300_a0.cl b/OpenCL/m13300_a0.cl
index 5ac204b..bd45d38 100644 (file)
--- a/OpenCL/m13300_a0.cl
+++ b/OpenCL/m13300_a0.cl
@@ -9,20 +9,20 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 4
 #define DGST_R2 3
 #define DGST_R3 2
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 __kernel void m13300_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m13300_a1.cl b/OpenCL/m13300_a1.cl
index 39421ec..8f1ec30 100644 (file)
--- a/OpenCL/m13300_a1.cl
+++ b/OpenCL/m13300_a1.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 4
 #define DGST_R2 3
 #define DGST_R3 2
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 __kernel void m13300_m04 (__global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __global bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max)
 {

diff --git a/OpenCL/m13300_a3.cl b/OpenCL/m13300_a3.cl
index dfeff7b..7109f7b 100644 (file)
--- a/OpenCL/m13300_a3.cl
+++ b/OpenCL/m13300_a3.cl
@@ -9,18 +9,18 @@
 
 #define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 4
 #define DGST_R2 3
 #define DGST_R3 2
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void m13300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global kernel_rule_t *rules_buf, __global comb_t *combs_buf, __constant u32x * words_buf_r, __global void *tmps, __global void *hooks, __global u32 *bitmaps_buf_s1_a, __global u32 *bitmaps_buf_s1_b, __global u32 *bitmaps_buf_s1_c, __global u32 *bitmaps_buf_s1_d, __global u32 *bitmaps_buf_s2_a, __global u32 *bitmaps_buf_s2_b, __global u32 *bitmaps_buf_s2_c, __global u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global digest_t *digests_buf, __global u32 *hashes_shown, __global salt_t *salt_bufs, __global void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset)
 {

diff --git a/OpenCL/m13400.cl b/OpenCL/m13400.cl
index e0ec6d9..6079f55 100644 (file)
--- a/OpenCL/m13400.cl
+++ b/OpenCL/m13400.cl
@@ -7,19 +7,19 @@
 
 #define _KEEPASS_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_twofish256.c"
+#include "inc_cipher_twofish256.cl"
 
 __constant u32 te0[256] =
 {

diff --git a/OpenCL/m13500_a0.cl b/OpenCL/m13500_a0.cl
index 9538b46..9339212 100644 (file)
--- a/OpenCL/m13500_a0.cl
+++ b/OpenCL/m13500_a0.cl
@@ -8,20 +8,20 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "include/rp_kernel.h"
-#include "OpenCL/rp.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_rp.h"
+#include "inc_rp.cl"
+#include "inc_simd.cl"
 
 void memcat64c_be (u32x block[16], const u32 offset, u32x carry[16])
 {

diff --git a/OpenCL/m13500_a1.cl b/OpenCL/m13500_a1.cl
index c1e3db1..b07279b 100644 (file)
--- a/OpenCL/m13500_a1.cl
+++ b/OpenCL/m13500_a1.cl
@@ -8,18 +8,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void memcat64c_be (u32x block[16], const u32 offset, u32x carry[16])
 {

diff --git a/OpenCL/m13500_a3.cl b/OpenCL/m13500_a3.cl
index 046db7e..28d7427 100644 (file)
--- a/OpenCL/m13500_a3.cl
+++ b/OpenCL/m13500_a3.cl
@@ -8,18 +8,18 @@
 //too much register pressure
 //#define NEW_SIMD_CODE
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 3
 #define DGST_R1 4
 #define DGST_R2 2
 #define DGST_R3 1
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
-#include "OpenCL/simd.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
+#include "inc_simd.cl"
 
 void memcat64c_be (u32x block[16], const u32 offset, u32x carry[16])
 {

diff --git a/OpenCL/m13600.cl b/OpenCL/m13600.cl
index 5499f40..5cd9c6d 100644 (file)
--- a/OpenCL/m13600.cl
+++ b/OpenCL/m13600.cl
@@ -5,21 +5,21 @@
 
 #define _ZIP2_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
+#include "inc_hash_functions.cl"
 
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#define COMPARE_S "OpenCL/check_single_comp4.c"
-#define COMPARE_M "OpenCL/check_multi_comp4.c"
+#define COMPARE_S "inc_comp_single.cl"
+#define COMPARE_M "inc_comp_multi.cl"
 
 void sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5])
 {

diff --git a/OpenCL/m13751.cl b/OpenCL/m13751.cl
index 06e3fab..f375323 100644 (file)
--- a/OpenCL/m13751.cl
+++ b/OpenCL/m13751.cl
@@ -5,21 +5,21 @@
 
 #define _SHA256_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m13752.cl b/OpenCL/m13752.cl
index 0613321..2f7a9a5 100644 (file)
--- a/OpenCL/m13752.cl
+++ b/OpenCL/m13752.cl
@@ -5,21 +5,21 @@
 
 #define _SHA256_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/m13753.cl b/OpenCL/m13753.cl
index be1a9e9..489901c 100644 (file)
--- a/OpenCL/m13753.cl
+++ b/OpenCL/m13753.cl
@@ -5,21 +5,21 @@
 
 #define _SHA256_
 
-#include "include/constants.h"
-#include "include/kernel_vendor.h"
+#include "inc_hash_constants.h"
+#include "inc_vendor.cl"
 
 #define DGST_R0 0
 #define DGST_R1 1
 #define DGST_R2 2
 #define DGST_R3 3
 
-#include "include/kernel_functions.c"
-#include "OpenCL/types_ocl.c"
-#include "OpenCL/common.c"
+#include "inc_hash_functions.cl"
+#include "inc_types.cl"
+#include "inc_common.cl"
 
-#include "OpenCL/kernel_aes256.c"
-#include "OpenCL/kernel_twofish256.c"
-#include "OpenCL/kernel_serpent256.c"
+#include "inc_cipher_aes256.cl"
+#include "inc_cipher_twofish256.cl"
+#include "inc_cipher_serpent256.cl"
 
 __constant u32 k_sha256[64] =
 {

diff --git a/OpenCL/markov_be.cl b/OpenCL/markov_be.cl
index a429b2e..9d0720f 100644 (file)
--- a/OpenCL/markov_be.cl
+++ b/OpenCL/markov_be.cl
@@ -3,11 +3,11 @@
  * License.....: MIT
  */
 
-#include "include/kernel_vendor.h"
+#include "inc_vendor.cl"
 
 #define CHARSIZ 256
 
-#include "OpenCL/types_ocl.c"
+#include "inc_types.cl"
 
 inline void generate_pw (u32 pw_buf[16], __global cs_t *root_css_buf, __global cs_t *markov_css_buf, const u32 pw_l_len, const u32 pw_r_len, const u32 mask80, const u32 bits14, const u32 bits15, u64 val)
 {

diff --git a/OpenCL/markov_le.cl b/OpenCL/markov_le.cl
index dc0001c..2729446 100644 (file)
--- a/OpenCL/markov_le.cl
+++ b/OpenCL/markov_le.cl
@@ -3,11 +3,11 @@
  * License.....: MIT
  */
 
-#include "include/kernel_vendor.h"
+#include "inc_vendor.cl"
 
 #define CHARSIZ 256
 
-#include "OpenCL/types_ocl.c"
+#include "inc_types.cl"
 
 inline void generate_pw (u32 pw_buf[16], __global cs_t *root_css_buf, __global cs_t *markov_css_buf, const u32 pw_l_len, const u32 pw_r_len, const u32 mask80, const u32 bits14, const u32 bits15, u64 val)
 {

diff --git a/OpenCL/rp.c b/OpenCL/rp.c
deleted file mode 100644 (file)
index 3a93e19..0000000
--- a/OpenCL/rp.c
+++ /dev/null
@@ -1,4357 +0,0 @@
-/**
- * Authors.....: Jens Steube <jens.steube@gmail.com>
- *               magnum <john.magnum@hushmail.com>
- *
- * License.....: MIT
- */
-
-inline u32  apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len);
-inline u32  apply_rules (const __global u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len);
-inline u32x apply_rules_vect (const u32 pw_buf0[4], const u32 pw_buf1[4], const u32 pw_len, const __global kernel_rule_t *rules_buf, const u32 il_pos, u32x w0[4], u32x w1[4]);
-
-inline u32 generate_cmask (u32 buf)
-{
-  const u32 rmask = ((buf & 0x40404040) >> 1)
-                 & ~((buf & 0x80808080) >> 2);
-
-  const u32 hmask = (buf & 0x1f1f1f1f) + 0x05050505;
-  const u32 lmask = (buf & 0x1f1f1f1f) + 0x1f1f1f1f;
-
-  return rmask & ~hmask & lmask;
-}
-
-inline void truncate_right (u32 w0[4], u32 w1[4], const u32 len)
-{
-  const u32 tmp = (1 << ((len % 4) * 8)) - 1;
-
-  switch (len / 4)
-  {
-    case  0:  w0[0] &= tmp;
-              w0[1]  = 0;
-              w0[2]  = 0;
-              w0[3]  = 0;
-              w1[0]  = 0;
-              w1[1]  = 0;
-              w1[2]  = 0;
-              w1[3]  = 0;
-              break;
-    case  1:  w0[1] &= tmp;
-              w0[2]  = 0;
-              w0[3]  = 0;
-              w1[0]  = 0;
-              w1[1]  = 0;
-              w1[2]  = 0;
-              w1[3]  = 0;
-              break;
-    case  2:  w0[2] &= tmp;
-              w0[3]  = 0;
-              w1[0]  = 0;
-              w1[1]  = 0;
-              w1[2]  = 0;
-              w1[3]  = 0;
-              break;
-    case  3:  w0[3] &= tmp;
-              w1[0]  = 0;
-              w1[1]  = 0;
-              w1[2]  = 0;
-              w1[3]  = 0;
-              break;
-    case  4:  w1[0] &= tmp;
-              w1[1]  = 0;
-              w1[2]  = 0;
-              w1[3]  = 0;
-              break;
-    case  5:  w1[1] &= tmp;
-              w1[2]  = 0;
-              w1[3]  = 0;
-              break;
-    case  6:  w1[2] &= tmp;
-              w1[3]  = 0;
-              break;
-    case  7:  w1[3] &= tmp;
-              break;
-  }
-}
-
-inline void truncate_left (u32 w0[4], u32 w1[4], const u32 len)
-{
-  const u32 tmp = ~((1 << ((len % 4) * 8)) - 1);
-
-  switch (len / 4)
-  {
-    case  0:  w0[0] &= tmp;
-              break;
-    case  1:  w0[0]  = 0;
-              w0[1] &= tmp;
-              break;
-    case  2:  w0[0]  = 0;
-              w0[1]  = 0;
-              w0[2] &= tmp;
-              break;
-    case  3:  w0[0]  = 0;
-              w0[1]  = 0;
-              w0[2]  = 0;
-              w0[3] &= tmp;
-              break;
-    case  4:  w0[0]  = 0;
-              w0[1]  = 0;
-              w0[2]  = 0;
-              w0[3]  = 0;
-              w1[0] &= tmp;
-              break;
-    case  5:  w0[0]  = 0;
-              w0[1]  = 0;
-              w0[2]  = 0;
-              w0[3]  = 0;
-              w1[0]  = 0;
-              w1[1] &= tmp;
-              break;
-    case  6:  w0[0]  = 0;
-              w0[1]  = 0;
-              w0[2]  = 0;
-              w0[3]  = 0;
-              w1[0]  = 0;
-              w1[1]  = 0;
-              w1[2] &= tmp;
-              break;
-    case  7:  w0[0]  = 0;
-              w0[1]  = 0;
-              w0[2]  = 0;
-              w0[3]  = 0;
-              w1[0]  = 0;
-              w1[1]  = 0;
-              w1[2]  = 0;
-              w1[3] &= tmp;
-              break;
-  }
-}
-
-inline void lshift_block (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4])
-{
-  #ifdef IS_NV
-  out0[0] = __byte_perm_S (in0[0], in0[1], 0x4321);
-  out0[1] = __byte_perm_S (in0[1], in0[2], 0x4321);
-  out0[2] = __byte_perm_S (in0[2], in0[3], 0x4321);
-  out0[3] = __byte_perm_S (in0[3], in1[0], 0x4321);
-  out1[0] = __byte_perm_S (in1[0], in1[1], 0x4321);
-  out1[1] = __byte_perm_S (in1[1], in1[2], 0x4321);
-  out1[2] = __byte_perm_S (in1[2], in1[3], 0x4321);
-  out1[3] = __byte_perm_S (in1[3],      0, 0x4321);
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  out0[0] = amd_bytealign_S (in0[1], in0[0], 1);
-  out0[1] = amd_bytealign_S (in0[2], in0[1], 1);
-  out0[2] = amd_bytealign_S (in0[3], in0[2], 1);
-  out0[3] = amd_bytealign_S (in1[0], in0[3], 1);
-  out1[0] = amd_bytealign_S (in1[1], in1[0], 1);
-  out1[1] = amd_bytealign_S (in1[2], in1[1], 1);
-  out1[2] = amd_bytealign_S (in1[3], in1[2], 1);
-  out1[3] = amd_bytealign_S (     0, in1[3], 1);
-  #endif
-}
-
-inline void rshift_block (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4])
-{
-  #ifdef IS_NV
-  out1[3] = __byte_perm_S (in1[2], in1[3], 0x6543);
-  out1[2] = __byte_perm_S (in1[1], in1[2], 0x6543);
-  out1[1] = __byte_perm_S (in1[0], in1[1], 0x6543);
-  out1[0] = __byte_perm_S (in0[3], in1[0], 0x6543);
-  out0[3] = __byte_perm_S (in0[2], in0[3], 0x6543);
-  out0[2] = __byte_perm_S (in0[1], in0[2], 0x6543);
-  out0[1] = __byte_perm_S (in0[0], in0[1], 0x6543);
-  out0[0] = __byte_perm_S (     0, in0[0], 0x6543);
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  out1[3] = amd_bytealign_S (in1[3], in1[2], 3);
-  out1[2] = amd_bytealign_S (in1[2], in1[1], 3);
-  out1[1] = amd_bytealign_S (in1[1], in1[0], 3);
-  out1[0] = amd_bytealign_S (in1[0], in0[3], 3);
-  out0[3] = amd_bytealign_S (in0[3], in0[2], 3);
-  out0[2] = amd_bytealign_S (in0[2], in0[1], 3);
-  out0[1] = amd_bytealign_S (in0[1], in0[0], 3);
-  out0[0] = amd_bytealign_S (in0[0],      0, 3);
-  #endif
-}
-
-inline void lshift_block_N (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4], const u32 num)
-{
-  #ifdef IS_NV
-  switch (num)
-  {
-    case  0:  out0[0] = in0[0];
-              out0[1] = in0[1];
-              out0[2] = in0[2];
-              out0[3] = in0[3];
-              out1[0] = in1[0];
-              out1[1] = in1[1];
-              out1[2] = in1[2];
-              out1[3] = in1[3];
-              break;
-    case  1:  out0[0] = __byte_perm_S (in0[0], in0[1], 0x4321);
-              out0[1] = __byte_perm_S (in0[1], in0[2], 0x4321);
-              out0[2] = __byte_perm_S (in0[2], in0[3], 0x4321);
-              out0[3] = __byte_perm_S (in0[3], in1[0], 0x4321);
-              out1[0] = __byte_perm_S (in1[0], in1[1], 0x4321);
-              out1[1] = __byte_perm_S (in1[1], in1[2], 0x4321);
-              out1[2] = __byte_perm_S (in1[2], in1[3], 0x4321);
-              out1[3] = __byte_perm_S (in1[3],      0, 0x4321);
-              break;
-    case  2:  out0[0] = __byte_perm_S (in0[0], in0[1], 0x5432);
-              out0[1] = __byte_perm_S (in0[1], in0[2], 0x5432);
-              out0[2] = __byte_perm_S (in0[2], in0[3], 0x5432);
-              out0[3] = __byte_perm_S (in0[3], in1[0], 0x5432);
-              out1[0] = __byte_perm_S (in1[0], in1[1], 0x5432);
-              out1[1] = __byte_perm_S (in1[1], in1[2], 0x5432);
-              out1[2] = __byte_perm_S (in1[2], in1[3], 0x5432);
-              out1[3] = __byte_perm_S (in1[3],      0, 0x5432);
-              break;
-    case  3:  out0[0] = __byte_perm_S (in0[0], in0[1], 0x6543);
-              out0[1] = __byte_perm_S (in0[1], in0[2], 0x6543);
-              out0[2] = __byte_perm_S (in0[2], in0[3], 0x6543);
-              out0[3] = __byte_perm_S (in0[3], in1[0], 0x6543);
-              out1[0] = __byte_perm_S (in1[0], in1[1], 0x6543);
-              out1[1] = __byte_perm_S (in1[1], in1[2], 0x6543);
-              out1[2] = __byte_perm_S (in1[2], in1[3], 0x6543);
-              out1[3] = __byte_perm_S (in1[3],      0, 0x6543);
-              break;
-    case  4:  out0[0] = in0[1];
-              out0[1] = in0[2];
-              out0[2] = in0[3];
-              out0[3] = in1[0];
-              out1[0] = in1[1];
-              out1[1] = in1[2];
-              out1[2] = in1[3];
-              out1[3] = 0;
-              break;
-    case  5:  out0[0] = __byte_perm_S (in0[1], in0[2], 0x4321);
-              out0[1] = __byte_perm_S (in0[2], in0[3], 0x4321);
-              out0[2] = __byte_perm_S (in0[3], in1[0], 0x4321);
-              out0[3] = __byte_perm_S (in1[0], in1[1], 0x4321);
-              out1[0] = __byte_perm_S (in1[1], in1[2], 0x4321);
-              out1[1] = __byte_perm_S (in1[2], in1[3], 0x4321);
-              out1[2] = __byte_perm_S (in1[3],      0, 0x4321);
-              out1[3] = 0;
-              break;
-    case  6:  out0[0] = __byte_perm_S (in0[1], in0[2], 0x5432);
-              out0[1] = __byte_perm_S (in0[2], in0[3], 0x5432);
-              out0[2] = __byte_perm_S (in0[3], in1[0], 0x5432);
-              out0[3] = __byte_perm_S (in1[0], in1[1], 0x5432);
-              out1[0] = __byte_perm_S (in1[1], in1[2], 0x5432);
-              out1[1] = __byte_perm_S (in1[2], in1[3], 0x5432);
-              out1[2] = __byte_perm_S (in1[3],      0, 0x5432);
-              out1[3] = 0;
-              break;
-    case  7:  out0[0] = __byte_perm_S (in0[1], in0[2], 0x6543);
-              out0[1] = __byte_perm_S (in0[2], in0[3], 0x6543);
-              out0[2] = __byte_perm_S (in0[3], in1[0], 0x6543);
-              out0[3] = __byte_perm_S (in1[0], in1[1], 0x6543);
-              out1[0] = __byte_perm_S (in1[1], in1[2], 0x6543);
-              out1[1] = __byte_perm_S (in1[2], in1[3], 0x6543);
-              out1[2] = __byte_perm_S (in1[3],      0, 0x6543);
-              out1[3] = 0;
-              break;
-    case  8:  out0[0] = in0[2];
-              out0[1] = in0[3];
-              out0[2] = in1[0];
-              out0[3] = in1[1];
-              out1[0] = in1[2];
-              out1[1] = in1[3];
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case  9:  out0[0] = __byte_perm_S (in0[2], in0[3], 0x4321);
-              out0[1] = __byte_perm_S (in0[3], in1[0], 0x4321);
-              out0[2] = __byte_perm_S (in1[0], in1[1], 0x4321);
-              out0[3] = __byte_perm_S (in1[1], in1[2], 0x4321);
-              out1[0] = __byte_perm_S (in1[2], in1[3], 0x4321);
-              out1[1] = __byte_perm_S (in1[3],      0, 0x4321);
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 10:  out0[0] = __byte_perm_S (in0[2], in0[3], 0x5432);
-              out0[1] = __byte_perm_S (in0[3], in1[0], 0x5432);
-              out0[2] = __byte_perm_S (in1[0], in1[1], 0x5432);
-              out0[3] = __byte_perm_S (in1[1], in1[2], 0x5432);
-              out1[0] = __byte_perm_S (in1[2], in1[3], 0x5432);
-              out1[1] = __byte_perm_S (in1[3],      0, 0x5432);
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 11:  out0[0] = __byte_perm_S (in0[2], in0[3], 0x6543);
-              out0[1] = __byte_perm_S (in0[3], in1[0], 0x6543);
-              out0[2] = __byte_perm_S (in1[0], in1[1], 0x6543);
-              out0[3] = __byte_perm_S (in1[1], in1[2], 0x6543);
-              out1[0] = __byte_perm_S (in1[2], in1[3], 0x6543);
-              out1[1] = __byte_perm_S (in1[3],      0, 0x6543);
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 12:  out0[0] = in0[3];
-              out0[1] = in1[0];
-              out0[2] = in1[1];
-              out0[3] = in1[2];
-              out1[0] = in1[3];
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 13:
-              out0[0] = __byte_perm_S (in0[3], in1[0], 0x4321);
-              out0[1] = __byte_perm_S (in1[0], in1[1], 0x4321);
-              out0[2] = __byte_perm_S (in1[1], in1[2], 0x4321);
-              out0[3] = __byte_perm_S (in1[2], in1[3], 0x4321);
-              out1[0] = __byte_perm_S (in1[3],      0, 0x4321);
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 14:  out0[0] = __byte_perm_S (in0[3], in1[0], 0x5432);
-              out0[1] = __byte_perm_S (in1[0], in1[1], 0x5432);
-              out0[2] = __byte_perm_S (in1[1], in1[2], 0x5432);
-              out0[3] = __byte_perm_S (in1[2], in1[3], 0x5432);
-              out1[0] = __byte_perm_S (in1[3],      0, 0x5432);
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 15:  out0[0] = __byte_perm_S (in0[3], in1[0], 0x6543);
-              out0[1] = __byte_perm_S (in1[0], in1[1], 0x6543);
-              out0[2] = __byte_perm_S (in1[1], in1[2], 0x6543);
-              out0[3] = __byte_perm_S (in1[2], in1[3], 0x6543);
-              out1[0] = __byte_perm_S (in1[3],      0, 0x6543);
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 16:  out0[0] = in1[0];
-              out0[1] = in1[1];
-              out0[2] = in1[2];
-              out0[3] = in1[3];
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 17:  out0[0] = __byte_perm_S (in1[0], in1[1], 0x4321);
-              out0[1] = __byte_perm_S (in1[1], in1[2], 0x4321);
-              out0[2] = __byte_perm_S (in1[2], in1[3], 0x4321);
-              out0[3] = __byte_perm_S (in1[3],      0, 0x4321);
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 18:  out0[0] = __byte_perm_S (in1[0], in1[1], 0x5432);
-              out0[1] = __byte_perm_S (in1[1], in1[2], 0x5432);
-              out0[2] = __byte_perm_S (in1[2], in1[3], 0x5432);
-              out0[3] = __byte_perm_S (in1[3],      0, 0x5432);
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 19:  out0[0] = __byte_perm_S (in1[0], in1[1], 0x6543);
-              out0[1] = __byte_perm_S (in1[1], in1[2], 0x6543);
-              out0[2] = __byte_perm_S (in1[2], in1[3], 0x6543);
-              out0[3] = __byte_perm_S (in1[3],      0, 0x6543);
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 20:  out0[0] = in1[1];
-              out0[1] = in1[2];
-              out0[2] = in1[3];
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 21:  out0[0] = __byte_perm_S (in1[1], in1[2], 0x4321);
-              out0[1] = __byte_perm_S (in1[2], in1[3], 0x4321);
-              out0[2] = __byte_perm_S (in1[3],      0, 0x4321);
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 22:  out0[0] = __byte_perm_S (in1[1], in1[2], 0x5432);
-              out0[1] = __byte_perm_S (in1[2], in1[3], 0x5432);
-              out0[2] = __byte_perm_S (in1[3],      0, 0x5432);
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 23:  out0[0] = __byte_perm_S (in1[1], in1[2], 0x6543);
-              out0[1] = __byte_perm_S (in1[2], in1[3], 0x6543);
-              out0[2] = __byte_perm_S (in1[3],      0, 0x6543);
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 24:  out0[0] = in1[2];
-              out0[1] = in1[3];
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 25:  out0[0] = __byte_perm_S (in1[2], in1[3], 0x4321);
-              out0[1] = __byte_perm_S (in1[3],      0, 0x4321);
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 26:  out0[0] = __byte_perm_S (in1[2], in1[3], 0x5432);
-              out0[1] = __byte_perm_S (in1[3],      0, 0x5432);
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 27:  out0[0] = __byte_perm_S (in1[2], in1[3], 0x6543);
-              out0[1] = __byte_perm_S (in1[3],      0, 0x6543);
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 28:  out0[0] = in1[3];
-              out0[1] = 0;
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 29:  out0[0] = __byte_perm_S (in1[3],     0, 0x4321);
-              out0[1] = 0;
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 30:  out0[0] =  __byte_perm_S (in1[3],     0, 0x5432);
-              out0[1] = 0;
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 31:  out0[0] =  __byte_perm_S (in1[3],     0, 0x6543);
-              out0[1] = 0;
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-  }
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  switch (num)
-  {
-    case  0:  out0[0] = in0[0];
-              out0[1] = in0[1];
-              out0[2] = in0[2];
-              out0[3] = in0[3];
-              out1[0] = in1[0];
-              out1[1] = in1[1];
-              out1[2] = in1[2];
-              out1[3] = in1[3];
-              break;
-    case  1:  out0[0] = amd_bytealign_S (in0[1], in0[0], 1);
-              out0[1] = amd_bytealign_S (in0[2], in0[1], 1);
-              out0[2] = amd_bytealign_S (in0[3], in0[2], 1);
-              out0[3] = amd_bytealign_S (in1[0], in0[3], 1);
-              out1[0] = amd_bytealign_S (in1[1], in1[0], 1);
-              out1[1] = amd_bytealign_S (in1[2], in1[1], 1);
-              out1[2] = amd_bytealign_S (in1[3], in1[2], 1);
-              out1[3] = amd_bytealign_S (     0, in1[3], 1);
-              break;
-    case  2:  out0[0] = amd_bytealign_S (in0[1], in0[0], 2);
-              out0[1] = amd_bytealign_S (in0[2], in0[1], 2);
-              out0[2] = amd_bytealign_S (in0[3], in0[2], 2);
-              out0[3] = amd_bytealign_S (in1[0], in0[3], 2);
-              out1[0] = amd_bytealign_S (in1[1], in1[0], 2);
-              out1[1] = amd_bytealign_S (in1[2], in1[1], 2);
-              out1[2] = amd_bytealign_S (in1[3], in1[2], 2);
-              out1[3] = amd_bytealign_S (     0, in1[3], 2);
-              break;
-    case  3:  out0[0] = amd_bytealign_S (in0[1], in0[0], 3);
-              out0[1] = amd_bytealign_S (in0[2], in0[1], 3);
-              out0[2] = amd_bytealign_S (in0[3], in0[2], 3);
-              out0[3] = amd_bytealign_S (in1[0], in0[3], 3);
-              out1[0] = amd_bytealign_S (in1[1], in1[0], 3);
-              out1[1] = amd_bytealign_S (in1[2], in1[1], 3);
-              out1[2] = amd_bytealign_S (in1[3], in1[2], 3);
-              out1[3] = amd_bytealign_S (     0, in1[3], 3);
-              break;
-    case  4:  out0[0] = in0[1];
-              out0[1] = in0[2];
-              out0[2] = in0[3];
-              out0[3] = in1[0];
-              out1[0] = in1[1];
-              out1[1] = in1[2];
-              out1[2] = in1[3];
-              out1[3] = 0;
-              break;
-    case  5:  out0[0] = amd_bytealign_S (in0[2], in0[1], 1);
-              out0[1] = amd_bytealign_S (in0[3], in0[2], 1);
-              out0[2] = amd_bytealign_S (in1[0], in0[3], 1);
-              out0[3] = amd_bytealign_S (in1[1], in1[0], 1);
-              out1[0] = amd_bytealign_S (in1[2], in1[1], 1);
-              out1[1] = amd_bytealign_S (in1[3], in1[2], 1);
-              out1[2] = amd_bytealign_S (     0, in1[3], 1);
-              out1[3] = 0;
-              break;
-    case  6:  out0[0] = amd_bytealign_S (in0[2], in0[1], 2);
-              out0[1] = amd_bytealign_S (in0[3], in0[2], 2);
-              out0[2] = amd_bytealign_S (in1[0], in0[3], 2);
-              out0[3] = amd_bytealign_S (in1[1], in1[0], 2);
-              out1[0] = amd_bytealign_S (in1[2], in1[1], 2);
-              out1[1] = amd_bytealign_S (in1[3], in1[2], 2);
-              out1[2] = amd_bytealign_S (     0, in1[3], 2);
-              out1[3] = 0;
-              break;
-    case  7:  out0[0] = amd_bytealign_S (in0[2], in0[1], 3);
-              out0[1] = amd_bytealign_S (in0[3], in0[2], 3);
-              out0[2] = amd_bytealign_S (in1[0], in0[3], 3);
-              out0[3] = amd_bytealign_S (in1[1], in1[0], 3);
-              out1[0] = amd_bytealign_S (in1[2], in1[1], 3);
-              out1[1] = amd_bytealign_S (in1[3], in1[2], 3);
-              out1[2] = amd_bytealign_S (     0, in1[3], 3);
-              out1[3] = 0;
-              break;
-    case  8:  out0[0] = in0[2];
-              out0[1] = in0[3];
-              out0[2] = in1[0];
-              out0[3] = in1[1];
-              out1[0] = in1[2];
-              out1[1] = in1[3];
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case  9:  out0[0] = amd_bytealign_S (in0[3], in0[2], 1);
-              out0[1] = amd_bytealign_S (in1[0], in0[3], 1);
-              out0[2] = amd_bytealign_S (in1[1], in1[0], 1);
-              out0[3] = amd_bytealign_S (in1[2], in1[1], 1);
-              out1[0] = amd_bytealign_S (in1[3], in1[2], 1);
-              out1[1] = amd_bytealign_S (     0, in1[3], 1);
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 10:  out0[0] = amd_bytealign_S (in0[3], in0[2], 2);
-              out0[1] = amd_bytealign_S (in1[0], in0[3], 2);
-              out0[2] = amd_bytealign_S (in1[1], in1[0], 2);
-              out0[3] = amd_bytealign_S (in1[2], in1[1], 2);
-              out1[0] = amd_bytealign_S (in1[3], in1[2], 2);
-              out1[1] = amd_bytealign_S (     0, in1[3], 2);
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 11:  out0[0] = amd_bytealign_S (in0[3], in0[2], 3);
-              out0[1] = amd_bytealign_S (in1[0], in0[3], 3);
-              out0[2] = amd_bytealign_S (in1[1], in1[0], 3);
-              out0[3] = amd_bytealign_S (in1[2], in1[1], 3);
-              out1[0] = amd_bytealign_S (in1[3], in1[2], 3);
-              out1[1] = amd_bytealign_S (     0, in1[3], 3);
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 12:  out0[0] = in0[3];
-              out0[1] = in1[0];
-              out0[2] = in1[1];
-              out0[3] = in1[2];
-              out1[0] = in1[3];
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 13:  out0[0] = amd_bytealign_S (in1[0], in0[3], 1);
-              out0[1] = amd_bytealign_S (in1[1], in1[0], 1);
-              out0[2] = amd_bytealign_S (in1[2], in1[1], 1);
-              out0[3] = amd_bytealign_S (in1[3], in1[2], 1);
-              out1[0] = amd_bytealign_S (     0, in1[3], 1);
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 14:  out0[0] = amd_bytealign_S (in1[0], in0[3], 2);
-              out0[1] = amd_bytealign_S (in1[1], in1[0], 2);
-              out0[2] = amd_bytealign_S (in1[2], in1[1], 2);
-              out0[3] = amd_bytealign_S (in1[3], in1[2], 2);
-              out1[0] = amd_bytealign_S (     0, in1[3], 2);
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 15:  out0[0] = amd_bytealign_S (in1[0], in0[3], 3);
-              out0[1] = amd_bytealign_S (in1[1], in1[0], 3);
-              out0[2] = amd_bytealign_S (in1[2], in1[1], 3);
-              out0[3] = amd_bytealign_S (in1[3], in1[2], 3);
-              out1[0] = amd_bytealign_S (     0, in1[3], 3);
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 16:  out0[0] = in1[0];
-              out0[1] = in1[1];
-              out0[2] = in1[2];
-              out0[3] = in1[3];
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 17:  out0[0] = amd_bytealign_S (in1[1], in1[0], 1);
-              out0[1] = amd_bytealign_S (in1[2], in1[1], 1);
-              out0[2] = amd_bytealign_S (in1[3], in1[2], 1);
-              out0[3] = amd_bytealign_S (     0, in1[3], 1);
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 18:  out0[0] = amd_bytealign_S (in1[1], in1[0], 2);
-              out0[1] = amd_bytealign_S (in1[2], in1[1], 2);
-              out0[2] = amd_bytealign_S (in1[3], in1[2], 2);
-              out0[3] = amd_bytealign_S (     0, in1[3], 2);
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 19:  out0[0] = amd_bytealign_S (in1[1], in1[0], 3);
-              out0[1] = amd_bytealign_S (in1[2], in1[1], 3);
-              out0[2] = amd_bytealign_S (in1[3], in1[2], 3);
-              out0[3] = amd_bytealign_S (     0, in1[3], 3);
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 20:  out0[0] = in1[1];
-              out0[1] = in1[2];
-              out0[2] = in1[3];
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 21:  out0[0] = amd_bytealign_S (in1[2], in1[1], 1);
-              out0[1] = amd_bytealign_S (in1[3], in1[2], 1);
-              out0[2] = amd_bytealign_S (     0, in1[3], 1);
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 22:  out0[0] = amd_bytealign_S (in1[2], in1[1], 2);
-              out0[1] = amd_bytealign_S (in1[3], in1[2], 2);
-              out0[2] = amd_bytealign_S (     0, in1[3], 2);
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 23:  out0[0] = amd_bytealign_S (in1[2], in1[1], 3);
-              out0[1] = amd_bytealign_S (in1[3], in1[2], 3);
-              out0[2] = amd_bytealign_S (     0, in1[3], 3);
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 24:  out0[0] = in1[2];
-              out0[1] = in1[3];
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 25:  out0[0] = amd_bytealign_S (in1[3], in1[2], 1);
-              out0[1] = amd_bytealign_S (     0, in1[3], 1);
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 26:  out0[0] = amd_bytealign_S (in1[3], in1[2], 2);
-              out0[1] = amd_bytealign_S (     0, in1[3], 2);
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 27:  out0[0] = amd_bytealign_S (in1[3], in1[2], 3);
-              out0[1] = amd_bytealign_S (     0, in1[3], 3);
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 28:  out0[0] = in1[3];
-              out0[1] = 0;
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 29:  out0[0] = amd_bytealign_S (     0, in1[3], 1);
-              out0[1] = 0;
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 30:  out0[0] = amd_bytealign_S (     0, in1[3], 2);
-              out0[1] = 0;
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-    case 31:  out0[0] = amd_bytealign_S (     0, in1[3], 3);
-              out0[1] = 0;
-              out0[2] = 0;
-              out0[3] = 0;
-              out1[0] = 0;
-              out1[1] = 0;
-              out1[2] = 0;
-              out1[3] = 0;
-              break;
-  }
-  #endif
-}
-
-inline void rshift_block_N (const u32 in0[4], const u32 in1[4], u32 out0[4], u32 out1[4], const u32 num)
-{
-  #ifdef IS_NV
-  switch (num)
-  {
-    case  0:  out1[3] = in1[3];
-              out1[2] = in1[2];
-              out1[1] = in1[1];
-              out1[0] = in1[0];
-              out0[3] = in0[3];
-              out0[2] = in0[2];
-              out0[1] = in0[1];
-              out0[0] = in0[0];
-              break;
-    case  1:  out1[3] = __byte_perm_S (in1[2], in1[3], 0x6543);
-              out1[2] = __byte_perm_S (in1[1], in1[2], 0x6543);
-              out1[1] = __byte_perm_S (in1[0], in1[1], 0x6543);
-              out1[0] = __byte_perm_S (in0[3], in1[0], 0x6543);
-              out0[3] = __byte_perm_S (in0[2], in0[3], 0x6543);
-              out0[2] = __byte_perm_S (in0[1], in0[2], 0x6543);
-              out0[1] = __byte_perm_S (in0[0], in0[1], 0x6543);
-              out0[0] = __byte_perm_S (     0, in0[0], 0x6543);
-              break;
-    case  2:  out1[3] = __byte_perm_S (in1[2], in1[3], 0x5432);
-              out1[2] = __byte_perm_S (in1[1], in1[2], 0x5432);
-              out1[1] = __byte_perm_S (in1[0], in1[1], 0x5432);
-              out1[0] = __byte_perm_S (in0[3], in1[0], 0x5432);
-              out0[3] = __byte_perm_S (in0[2], in0[3], 0x5432);
-              out0[2] = __byte_perm_S (in0[1], in0[2], 0x5432);
-              out0[1] = __byte_perm_S (in0[0], in0[1], 0x5432);
-              out0[0] = __byte_perm_S (     0, in0[0], 0x5432);
-              break;
-    case  3:  out1[3] = __byte_perm_S (in1[2], in1[3], 0x4321);
-              out1[2] = __byte_perm_S (in1[1], in1[2], 0x4321);
-              out1[1] = __byte_perm_S (in1[0], in1[1], 0x4321);
-              out1[0] = __byte_perm_S (in0[3], in1[0], 0x4321);
-              out0[3] = __byte_perm_S (in0[2], in0[3], 0x4321);
-              out0[2] = __byte_perm_S (in0[1], in0[2], 0x4321);
-              out0[1] = __byte_perm_S (in0[0], in0[1], 0x4321);
-              out0[0] = __byte_perm_S (     0, in0[0], 0x4321);
-              break;
-    case  4:  out1[3] = in1[2];
-              out1[2] = in1[1];
-              out1[1] = in1[0];
-              out1[0] = in0[3];
-              out0[3] = in0[2];
-              out0[2] = in0[1];
-              out0[1] = in0[0];
-              out0[0] = 0;
-              break;
-    case  5:  out1[3] = __byte_perm_S (in1[1], in1[2], 0x6543);
-              out1[2] = __byte_perm_S (in1[0], in1[1], 0x6543);
-              out1[1] = __byte_perm_S (in0[3], in1[0], 0x6543);
-              out1[0] = __byte_perm_S (in0[2], in0[3], 0x6543);
-              out0[3] = __byte_perm_S (in0[1], in0[2], 0x6543);
-              out0[2] = __byte_perm_S (in0[0], in0[1], 0x6543);
-              out0[1] = __byte_perm_S (     0, in0[0], 0x6543);
-              out0[0] = 0;
-              break;
-    case  6:  out1[3] = __byte_perm_S (in1[1], in1[2], 0x5432);
-              out1[2] = __byte_perm_S (in1[0], in1[1], 0x5432);
-              out1[1] = __byte_perm_S (in0[3], in1[0], 0x5432);
-              out1[0] = __byte_perm_S (in0[2], in0[3], 0x5432);
-              out0[3] = __byte_perm_S (in0[1], in0[2], 0x5432);
-              out0[2] = __byte_perm_S (in0[0], in0[1], 0x5432);
-              out0[1] = __byte_perm_S (     0, in0[0], 0x5432);
-              out0[0] = 0;
-              break;
-    case  7:  out1[3] = __byte_perm_S (in1[1], in1[2], 0x4321);
-              out1[2] = __byte_perm_S (in1[0], in1[1], 0x4321);
-              out1[1] = __byte_perm_S (in0[3], in1[0], 0x4321);
-              out1[0] = __byte_perm_S (in0[2], in0[3], 0x4321);
-              out0[3] = __byte_perm_S (in0[1], in0[2], 0x4321);
-              out0[2] = __byte_perm_S (in0[0], in0[1], 0x4321);
-              out0[1] = __byte_perm_S (     0, in0[0], 0x4321);
-              out0[0] = 0;
-              break;
-    case  8:  out1[3] = in1[1];
-              out1[2] = in1[0];
-              out1[1] = in0[3];
-              out1[0] = in0[2];
-              out0[3] = in0[1];
-              out0[2] = in0[0];
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case  9:  out1[3] = __byte_perm_S (in1[0], in1[1], 0x6543);
-              out1[2] = __byte_perm_S (in0[3], in1[0], 0x6543);
-              out1[1] = __byte_perm_S (in0[2], in0[3], 0x6543);
-              out1[0] = __byte_perm_S (in0[1], in0[2], 0x6543);
-              out0[3] = __byte_perm_S (in0[0], in0[1], 0x6543);
-              out0[2] = __byte_perm_S (     0, in0[0], 0x6543);
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 10:  out1[3] = __byte_perm_S (in1[0], in1[1], 0x5432);
-              out1[2] = __byte_perm_S (in0[3], in1[0], 0x5432);
-              out1[1] = __byte_perm_S (in0[2], in0[3], 0x5432);
-              out1[0] = __byte_perm_S (in0[1], in0[2], 0x5432);
-              out0[3] = __byte_perm_S (in0[0], in0[1], 0x5432);
-              out0[2] = __byte_perm_S (     0, in0[0], 0x5432);
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 11:  out1[3] = __byte_perm_S (in1[0], in1[1], 0x4321);
-              out1[2] = __byte_perm_S (in0[3], in1[0], 0x4321);
-              out1[1] = __byte_perm_S (in0[2], in0[3], 0x4321);
-              out1[0] = __byte_perm_S (in0[1], in0[2], 0x4321);
-              out0[3] = __byte_perm_S (in0[0], in0[1], 0x4321);
-              out0[2] = __byte_perm_S (     0, in0[0], 0x4321);
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 12:  out1[3] = in1[0];
-              out1[2] = in0[3];
-              out1[1] = in0[2];
-              out1[0] = in0[1];
-              out0[3] = in0[0];
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 13:  out1[3] = __byte_perm_S (in0[3], in1[0], 0x6543);
-              out1[2] = __byte_perm_S (in0[2], in0[3], 0x6543);
-              out1[1] = __byte_perm_S (in0[1], in0[2], 0x6543);
-              out1[0] = __byte_perm_S (in0[0], in0[1], 0x6543);
-              out0[3] = __byte_perm_S (     0, in0[0], 0x6543);
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 14:  out1[3] = __byte_perm_S (in0[3], in1[0], 0x5432);
-              out1[2] = __byte_perm_S (in0[2], in0[3], 0x5432);
-              out1[1] = __byte_perm_S (in0[1], in0[2], 0x5432);
-              out1[0] = __byte_perm_S (in0[0], in0[1], 0x5432);
-              out0[3] = __byte_perm_S (     0, in0[0], 0x5432);
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 15:  out1[3] = __byte_perm_S (in0[3], in1[0], 0x4321);
-              out1[2] = __byte_perm_S (in0[2], in0[3], 0x4321);
-              out1[1] = __byte_perm_S (in0[1], in0[2], 0x4321);
-              out1[0] = __byte_perm_S (in0[0], in0[1], 0x4321);
-              out0[3] = __byte_perm_S (     0, in0[0], 0x4321);
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 16:  out1[3] = in0[3];
-              out1[2] = in0[2];
-              out1[1] = in0[1];
-              out1[0] = in0[0];
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 17:  out1[3] = __byte_perm_S (in0[2], in0[3], 0x6543);
-              out1[2] = __byte_perm_S (in0[1], in0[2], 0x6543);
-              out1[1] = __byte_perm_S (in0[0], in0[1], 0x6543);
-              out1[0] = __byte_perm_S (     0, in0[0], 0x6543);
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 18:  out1[3] = __byte_perm_S (in0[2], in0[3], 0x5432);
-              out1[2] = __byte_perm_S (in0[1], in0[2], 0x5432);
-              out1[1] = __byte_perm_S (in0[0], in0[1], 0x5432);
-              out1[0] = __byte_perm_S (     0, in0[0], 0x5432);
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 19:  out1[3] = __byte_perm_S (in0[2], in0[3], 0x4321);
-              out1[2] = __byte_perm_S (in0[1], in0[2], 0x4321);
-              out1[1] = __byte_perm_S (in0[0], in0[1], 0x4321);
-              out1[0] = __byte_perm_S (     0, in0[0], 0x4321);
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 20:  out1[3] = in0[2];
-              out1[2] = in0[1];
-              out1[1] = in0[0];
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 21:  out1[3] = __byte_perm_S (in0[1], in0[2], 0x6543);
-              out1[2] = __byte_perm_S (in0[0], in0[1], 0x6543);
-              out1[1] = __byte_perm_S (     0, in0[0], 0x6543);
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 22:  out1[3] = __byte_perm_S (in0[1], in0[2], 0x5432);
-              out1[2] = __byte_perm_S (in0[0], in0[1], 0x5432);
-              out1[1] = __byte_perm_S (     0, in0[0], 0x5432);
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 23:  out1[3] = __byte_perm_S (in0[1], in0[2], 0x4321);
-              out1[2] = __byte_perm_S (in0[0], in0[1], 0x4321);
-              out1[1] = __byte_perm_S (     0, in0[0], 0x4321);
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 24:  out1[3] = in0[1];
-              out1[2] = in0[0];
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 25:  out1[3] = __byte_perm_S (in0[0], in0[1], 0x6543);
-              out1[2] = __byte_perm_S (     0, in0[0], 0x6543);
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 26:  out1[3] = __byte_perm_S (in0[0], in0[1], 0x5432);
-              out1[2] = __byte_perm_S (     0, in0[0], 0x5432);
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 27:  out1[3] = __byte_perm_S (in0[0], in0[1], 0x4321);
-              out1[2] = __byte_perm_S (     0, in0[0], 0x4321);
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 28:  out1[3] = in0[0];
-              out1[2] = 0;
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 29:  out1[3] = __byte_perm_S (     0, in0[0], 0x6543);
-              out1[2] = 0;
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 30:  out1[3] = __byte_perm_S (     0, in0[0], 0x5432);
-              out1[2] = 0;
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 31:  out1[3] = __byte_perm_S (     0, in0[0], 0x4321);
-              out1[2] = 0;
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-  }
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  switch (num)
-  {
-    case  0:  out1[3] = in1[3];
-              out1[2] = in1[2];
-              out1[1] = in1[1];
-              out1[0] = in1[0];
-              out0[3] = in0[3];
-              out0[2] = in0[2];
-              out0[1] = in0[1];
-              out0[0] = in0[0];
-              break;
-    case  1:  out1[3] = amd_bytealign_S (in1[3], in1[2], 3);
-              out1[2] = amd_bytealign_S (in1[2], in1[1], 3);
-              out1[1] = amd_bytealign_S (in1[1], in1[0], 3);
-              out1[0] = amd_bytealign_S (in1[0], in0[3], 3);
-              out0[3] = amd_bytealign_S (in0[3], in0[2], 3);
-              out0[2] = amd_bytealign_S (in0[2], in0[1], 3);
-              out0[1] = amd_bytealign_S (in0[1], in0[0], 3);
-              out0[0] = amd_bytealign_S (in0[0],      0, 3);
-              break;
-    case  2:  out1[3] = amd_bytealign_S (in1[3], in1[2], 2);
-              out1[2] = amd_bytealign_S (in1[2], in1[1], 2);
-              out1[1] = amd_bytealign_S (in1[1], in1[0], 2);
-              out1[0] = amd_bytealign_S (in1[0], in0[3], 2);
-              out0[3] = amd_bytealign_S (in0[3], in0[2], 2);
-              out0[2] = amd_bytealign_S (in0[2], in0[1], 2);
-              out0[1] = amd_bytealign_S (in0[1], in0[0], 2);
-              out0[0] = amd_bytealign_S (in0[0],      0, 2);
-              break;
-    case  3:  out1[3] = amd_bytealign_S (in1[3], in1[2], 1);
-              out1[2] = amd_bytealign_S (in1[2], in1[1], 1);
-              out1[1] = amd_bytealign_S (in1[1], in1[0], 1);
-              out1[0] = amd_bytealign_S (in1[0], in0[3], 1);
-              out0[3] = amd_bytealign_S (in0[3], in0[2], 1);
-              out0[2] = amd_bytealign_S (in0[2], in0[1], 1);
-              out0[1] = amd_bytealign_S (in0[1], in0[0], 1);
-              out0[0] = amd_bytealign_S (in0[0],      0, 1);
-              break;
-    case  4:  out1[3] = in1[2];
-              out1[2] = in1[1];
-              out1[1] = in1[0];
-              out1[0] = in0[3];
-              out0[3] = in0[2];
-              out0[2] = in0[1];
-              out0[1] = in0[0];
-              out0[0] = 0;
-              break;
-    case  5:  out1[3] = amd_bytealign_S (in1[2], in1[1], 3);
-              out1[2] = amd_bytealign_S (in1[1], in1[0], 3);
-              out1[1] = amd_bytealign_S (in1[0], in0[3], 3);
-              out1[0] = amd_bytealign_S (in0[3], in0[2], 3);
-              out0[3] = amd_bytealign_S (in0[2], in0[1], 3);
-              out0[2] = amd_bytealign_S (in0[1], in0[0], 3);
-              out0[1] = amd_bytealign_S (in0[0],      0, 3);
-              out0[0] = 0;
-              break;
-    case  6:  out1[3] = amd_bytealign_S (in1[2], in1[1], 2);
-              out1[2] = amd_bytealign_S (in1[1], in1[0], 2);
-              out1[1] = amd_bytealign_S (in1[0], in0[3], 2);
-              out1[0] = amd_bytealign_S (in0[3], in0[2], 2);
-              out0[3] = amd_bytealign_S (in0[2], in0[1], 2);
-              out0[2] = amd_bytealign_S (in0[1], in0[0], 2);
-              out0[1] = amd_bytealign_S (in0[0],      0, 2);
-              out0[0] = 0;
-              break;
-    case  7:  out1[3] = amd_bytealign_S (in1[2], in1[1], 1);
-              out1[2] = amd_bytealign_S (in1[1], in1[0], 1);
-              out1[1] = amd_bytealign_S (in1[0], in0[3], 1);
-              out1[0] = amd_bytealign_S (in0[3], in0[2], 1);
-              out0[3] = amd_bytealign_S (in0[2], in0[1], 1);
-              out0[2] = amd_bytealign_S (in0[1], in0[0], 1);
-              out0[1] = amd_bytealign_S (in0[0],      0, 1);
-              out0[0] = 0;
-              break;
-    case  8:  out1[3] = in1[1];
-              out1[2] = in1[0];
-              out1[1] = in0[3];
-              out1[0] = in0[2];
-              out0[3] = in0[1];
-              out0[2] = in0[0];
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case  9:  out1[3] = amd_bytealign_S (in1[1], in1[0], 3);
-              out1[2] = amd_bytealign_S (in1[0], in0[3], 3);
-              out1[1] = amd_bytealign_S (in0[3], in0[2], 3);
-              out1[0] = amd_bytealign_S (in0[2], in0[1], 3);
-              out0[3] = amd_bytealign_S (in0[1], in0[0], 3);
-              out0[2] = amd_bytealign_S (in0[0],      0, 3);
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 10:  out1[3] = amd_bytealign_S (in1[1], in1[0], 2);
-              out1[2] = amd_bytealign_S (in1[0], in0[3], 2);
-              out1[1] = amd_bytealign_S (in0[3], in0[2], 2);
-              out1[0] = amd_bytealign_S (in0[2], in0[1], 2);
-              out0[3] = amd_bytealign_S (in0[1], in0[0], 2);
-              out0[2] = amd_bytealign_S (in0[0],      0, 2);
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 11:  out1[3] = amd_bytealign_S (in1[1], in1[0], 1);
-              out1[2] = amd_bytealign_S (in1[0], in0[3], 1);
-              out1[1] = amd_bytealign_S (in0[3], in0[2], 1);
-              out1[0] = amd_bytealign_S (in0[2], in0[1], 1);
-              out0[3] = amd_bytealign_S (in0[1], in0[0], 1);
-              out0[2] = amd_bytealign_S (in0[0],      0, 1);
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 12:  out1[3] = in1[0];
-              out1[2] = in0[3];
-              out1[1] = in0[2];
-              out1[0] = in0[1];
-              out0[3] = in0[0];
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 13:  out1[3] = amd_bytealign_S (in1[0], in0[3], 3);
-              out1[2] = amd_bytealign_S (in0[3], in0[2], 3);
-              out1[1] = amd_bytealign_S (in0[2], in0[1], 3);
-              out1[0] = amd_bytealign_S (in0[1], in0[0], 3);
-              out0[3] = amd_bytealign_S (in0[0],      0, 3);
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 14:  out1[3] = amd_bytealign_S (in1[0], in0[3], 2);
-              out1[2] = amd_bytealign_S (in0[3], in0[2], 2);
-              out1[1] = amd_bytealign_S (in0[2], in0[1], 2);
-              out1[0] = amd_bytealign_S (in0[1], in0[0], 2);
-              out0[3] = amd_bytealign_S (in0[0],      0, 2);
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 15:  out1[3] = amd_bytealign_S (in1[0], in0[3], 1);
-              out1[2] = amd_bytealign_S (in0[3], in0[2], 1);
-              out1[1] = amd_bytealign_S (in0[2], in0[1], 1);
-              out1[0] = amd_bytealign_S (in0[1], in0[0], 1);
-              out0[3] = amd_bytealign_S (in0[0],      0, 1);
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 16:  out1[3] = in0[3];
-              out1[2] = in0[2];
-              out1[1] = in0[1];
-              out1[0] = in0[0];
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 17:  out1[3] = amd_bytealign_S (in0[3], in0[2], 3);
-              out1[2] = amd_bytealign_S (in0[2], in0[1], 3);
-              out1[1] = amd_bytealign_S (in0[1], in0[0], 3);
-              out1[0] = amd_bytealign_S (in0[0],      0, 3);
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 18:  out1[3] = amd_bytealign_S (in0[3], in0[2], 2);
-              out1[2] = amd_bytealign_S (in0[2], in0[1], 2);
-              out1[1] = amd_bytealign_S (in0[1], in0[0], 2);
-              out1[0] = amd_bytealign_S (in0[0],      0, 2);
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 19:  out1[3] = amd_bytealign_S (in0[3], in0[2], 1);
-              out1[2] = amd_bytealign_S (in0[2], in0[1], 1);
-              out1[1] = amd_bytealign_S (in0[1], in0[0], 1);
-              out1[0] = amd_bytealign_S (in0[0],      0, 1);
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 20:  out1[3] = in0[2];
-              out1[2] = in0[1];
-              out1[1] = in0[0];
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 21:  out1[3] = amd_bytealign_S (in0[2], in0[1], 3);
-              out1[2] = amd_bytealign_S (in0[1], in0[0], 3);
-              out1[1] = amd_bytealign_S (in0[0],      0, 3);
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 22:  out1[3] = amd_bytealign_S (in0[2], in0[1], 2);
-              out1[2] = amd_bytealign_S (in0[1], in0[0], 2);
-              out1[1] = amd_bytealign_S (in0[0],      0, 2);
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 23:  out1[3] = amd_bytealign_S (in0[2], in0[1], 1);
-              out1[2] = amd_bytealign_S (in0[1], in0[0], 1);
-              out1[1] = amd_bytealign_S (in0[0],      0, 1);
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 24:  out1[3] = in0[1];
-              out1[2] = in0[0];
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 25:  out1[3] = amd_bytealign_S (in0[1], in0[0], 3);
-              out1[2] = amd_bytealign_S (in0[0],      0, 3);
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 26:  out1[3] = amd_bytealign_S (in0[1], in0[0], 2);
-              out1[2] = amd_bytealign_S (in0[0],      0, 2);
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 27:  out1[3] = amd_bytealign_S (in0[1], in0[0], 1);
-              out1[2] = amd_bytealign_S (in0[0],      0, 1);
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 28:  out1[3] = in0[0];
-              out1[2] = 0;
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 29:  out1[3] = amd_bytealign_S (in0[0],      0, 3);
-              out1[2] = 0;
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 30:  out1[3] = amd_bytealign_S (in0[0],      0, 2);
-              out1[2] = 0;
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-    case 31:  out1[3] = amd_bytealign_S (in0[0],      0, 1);
-              out1[2] = 0;
-              out1[1] = 0;
-              out1[0] = 0;
-              out0[3] = 0;
-              out0[2] = 0;
-              out0[1] = 0;
-              out0[0] = 0;
-              break;
-  }
-  #endif
-}
-
-inline void append_block1 (const u32 offset, u32 dst0[4], u32 dst1[4], const u32 src_r0)
-{
-  u32 tmp[2];
-
-  switch (offset & 3)
-  {
-    case  0:  tmp[0] = src_r0;
-              tmp[1] = 0;
-              break;
-    case  1:  tmp[0] = src_r0 <<  8;
-              tmp[1] = src_r0 >> 24;
-              break;
-    case  2:  tmp[0] = src_r0 << 16;
-              tmp[1] = src_r0 >> 16;
-              break;
-    case  3:  tmp[0] = src_r0 << 24;
-              tmp[1] = src_r0 >>  8;
-              break;
-  }
-
-  switch (offset / 4)
-  {
-    case  0:  dst0[0] |= tmp[0];
-              dst0[1]  = tmp[1];
-              break;
-    case  1:  dst0[1] |= tmp[0];
-              dst0[2]  = tmp[1];
-              break;
-    case  2:  dst0[2] |= tmp[0];
-              dst0[3]  = tmp[1];
-              break;
-    case  3:  dst0[3] |= tmp[0];
-              dst1[0]  = tmp[1];
-              break;
-    case  4:  dst1[0] |= tmp[0];
-              dst1[1]  = tmp[1];
-              break;
-    case  5:  dst1[1] |= tmp[0];
-              dst1[2]  = tmp[1];
-              break;
-    case  6:  dst1[2] |= tmp[0];
-              dst1[3]  = tmp[1];
-              break;
-    case  7:  dst1[3] |= tmp[0];
-              break;
-  }
-}
-
-inline void append_block8 (const u32 offset, u32 dst0[4], u32 dst1[4], const u32 src_l0[4], const u32 src_l1[4], const u32 src_r0[4], const u32 src_r1[4])
-{
-/*
-  #ifdef IS_NV
-  switch (offset)
-  {
-    case 0:
-      dst0[0] = src_r0[0];
-      dst0[1] = src_r0[1];
-      dst0[2] = src_r0[2];
-      dst0[3] = src_r0[3];
-      dst1[0] = src_r1[0];
-      dst1[1] = src_r1[1];
-      dst1[2] = src_r1[2];
-      dst1[3] = src_r1[3];
-      break;
-
-    case 1:
-      dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x6540);
-      dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
-      dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
-      dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
-      dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543);
-      dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543);
-      dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x6543);
-      dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x6543);
-      break;
-
-    case 2:
-      dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x5410);
-      dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
-      dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
-      dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
-      dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432);
-      dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432);
-      dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x5432);
-      dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x5432);
-      break;
-
-    case 3:
-      dst0[0] = __byte_perm_S (src_l0[0], src_r0[0], 0x4210);
-      dst0[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
-      dst0[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
-      dst0[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
-      dst1[0] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321);
-      dst1[1] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321);
-      dst1[2] = __byte_perm_S (src_r1[1], src_r1[2], 0x4321);
-      dst1[3] = __byte_perm_S (src_r1[2], src_r1[3], 0x4321);
-      break;
-
-    case 4:
-      dst0[1] = src_r0[0];
-      dst0[2] = src_r0[1];
-      dst0[3] = src_r0[2];
-      dst1[0] = src_r0[3];
-      dst1[1] = src_r1[0];
-      dst1[2] = src_r1[1];
-      dst1[3] = src_r1[2];
-      break;
-
-    case 5:
-      dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x6540);
-      dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
-      dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
-      dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
-      dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543);
-      dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543);
-      dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x6543);
-      break;
-
-    case 6:
-      dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x5410);
-      dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
-      dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
-      dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
-      dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432);
-      dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432);
-      dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x5432);
-      break;
-
-    case 7:
-      dst0[1] = __byte_perm_S (src_l0[1], src_r0[0], 0x4210);
-      dst0[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
-      dst0[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
-      dst1[0] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
-      dst1[1] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321);
-      dst1[2] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321);
-      dst1[3] = __byte_perm_S (src_r1[1], src_r1[2], 0x4321);
-      break;
-
-    case 8:
-      dst0[2] = src_r0[0];
-      dst0[3] = src_r0[1];
-      dst1[0] = src_r0[2];
-      dst1[1] = src_r0[3];
-      dst1[2] = src_r1[0];
-      dst1[3] = src_r1[1];
-      break;
-
-    case 9:
-      dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x6540);
-      dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
-      dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
-      dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
-      dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543);
-      dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x6543);
-      break;
-
-    case 10:
-      dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x5410);
-      dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
-      dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
-      dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
-      dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432);
-      dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x5432);
-      break;
-
-    case 11:
-      dst0[2] = __byte_perm_S (src_l0[2], src_r0[0], 0x4210);
-      dst0[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
-      dst1[0] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
-      dst1[1] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
-      dst1[2] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321);
-      dst1[3] = __byte_perm_S (src_r1[0], src_r1[1], 0x4321);
-      break;
-
-    case 12:
-      dst0[3] = src_r0[0];
-      dst1[0] = src_r0[1];
-      dst1[1] = src_r0[2];
-      dst1[2] = src_r0[3];
-      dst1[3] = src_r1[0];
-      break;
-
-    case 13:
-      dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x6540);
-      dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
-      dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
-      dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
-      dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x6543);
-      break;
-
-    case 14:
-      dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x5410);
-      dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
-      dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
-      dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
-      dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x5432);
-      break;
-
-    case 15:
-      dst0[3] = __byte_perm_S (src_l0[3], src_r0[0], 0x4210);
-      dst1[0] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
-      dst1[1] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
-      dst1[2] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
-      dst1[3] = __byte_perm_S (src_r0[3], src_r1[0], 0x4321);
-      break;
-
-    case 16:
-      dst1[0] = src_r0[0];
-      dst1[1] = src_r0[1];
-      dst1[2] = src_r0[2];
-      dst1[3] = src_r0[3];
-      break;
-
-    case 17:
-      dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x6540);
-      dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
-      dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
-      dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x6543);
-      break;
-
-    case 18:
-      dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x5410);
-      dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
-      dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
-      dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x5432);
-      break;
-
-    case 19:
-      dst1[0] = __byte_perm_S (src_l1[0], src_r0[0], 0x4210);
-      dst1[1] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
-      dst1[2] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
-      dst1[3] = __byte_perm_S (src_r0[2], src_r0[3], 0x4321);
-      break;
-
-    case 20:
-      dst1[1] = src_r0[0];
-      dst1[2] = src_r0[1];
-      dst1[3] = src_r0[2];
-      break;
-
-    case 21:
-      dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x6540);
-      dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
-      dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x6543);
-      break;
-
-    case 22:
-      dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x5410);
-      dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
-      dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x5432);
-      break;
-
-    case 23:
-      dst1[1] = __byte_perm_S (src_l1[1], src_r0[0], 0x4210);
-      dst1[2] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
-      dst1[3] = __byte_perm_S (src_r0[1], src_r0[2], 0x4321);
-      break;
-
-    case 24:
-      dst1[2] = src_r0[0];
-      dst1[3] = src_r0[1];
-      break;
-
-    case 25:
-      dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x6540);
-      dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x6543);
-      break;
-
-    case 26:
-      dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x5410);
-      dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x5432);
-      break;
-
-    case 27:
-      dst1[2] = __byte_perm_S (src_l1[2], src_r0[0], 0x4210);
-      dst1[3] = __byte_perm_S (src_r0[0], src_r0[1], 0x4321);
-      break;
-
-    case 28:
-      dst1[3] = src_r0[0];
-      break;
-
-    case 29:
-      dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x6540);
-      break;
-
-    case 30:
-      dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x5410);
-      break;
-
-    case 31:
-      dst1[3] = __byte_perm_S (src_l1[3], src_r0[0], 0x4210);
-      break;
-  }
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-*/
-  switch (offset)
-  {
-    case 31:
-      dst1[3] = src_l1[3] | src_r0[0] << 24;
-      break;
-    case 30:
-      dst1[3] = src_l1[3] | src_r0[0] << 16;
-      break;
-    case 29:
-      dst1[3] = src_l1[3] | src_r0[0] <<  8;
-      break;
-    case 28:
-      dst1[3] = src_r0[0];
-      break;
-    case 27:
-      dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
-      dst1[2] = src_l1[2] | src_r0[0] << 24;
-      break;
-    case 26:
-      dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
-      dst1[2] = src_l1[2] | src_r0[0] << 16;
-      break;
-    case 25:
-      dst1[3] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
-      dst1[2] = src_l1[2] | src_r0[0] <<  8;
-      break;
-    case 24:
-      dst1[3] = src_r0[1];
-      dst1[2] = src_r0[0];
-      break;
-    case 23:
-      dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
-      dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
-      dst1[1] = src_l1[1] | src_r0[0] << 24;
-      break;
-    case 22:
-      dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
-      dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
-      dst1[1] = src_l1[1] | src_r0[0] << 16;
-      break;
-    case 21:
-      dst1[3] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
-      dst1[2] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
-      dst1[1] = src_l1[1] | src_r0[0] <<  8;
-      break;
-    case 20:
-      dst1[3] = src_r0[2];
-      dst1[2] = src_r0[1];
-      dst1[1] = src_r0[0];
-      break;
-    case 19:
-      dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
-      dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
-      dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
-      dst1[0] = src_l1[0] | src_r0[0] << 24;
-      break;
-    case 18:
-      dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
-      dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
-      dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
-      dst1[0] = src_l1[0] | src_r0[0] << 16;
-      break;
-    case 17:
-      dst1[3] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
-      dst1[2] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
-      dst1[1] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
-      dst1[0] = src_l1[0] | src_r0[0] <<  8;
-      break;
-    case 16:
-      dst1[3] = src_r0[3];
-      dst1[2] = src_r0[2];
-      dst1[1] = src_r0[1];
-      dst1[0] = src_r0[0];
-      break;
-    case 15:
-      dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 1);
-      dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
-      dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
-      dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
-      dst0[3] = src_l0[3] | src_r0[0] << 24;
-      break;
-    case 14:
-      dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 2);
-      dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
-      dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
-      dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
-      dst0[3] = src_l0[3] | src_r0[0] << 16;
-      break;
-    case 13:
-      dst1[3] = amd_bytealign_S (src_r1[0], src_r0[3], 3);
-      dst1[2] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
-      dst1[1] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
-      dst1[0] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
-      dst0[3] = src_l0[3] | src_r0[0] <<  8;
-      break;
-    case 12:
-      dst1[3] = src_r1[0];
-      dst1[2] = src_r0[3];
-      dst1[1] = src_r0[2];
-      dst1[0] = src_r0[1];
-      dst0[3] = src_r0[0];
-      break;
-    case 11:
-      dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 1);
-      dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 1);
-      dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
-      dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
-      dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
-      dst0[2] = src_l0[2] | src_r0[0] << 24;
-      break;
-    case 10:
-      dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 2);
-      dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 2);
-      dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
-      dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
-      dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
-      dst0[2] = src_l0[2] | src_r0[0] << 16;
-      break;
-    case 9:
-      dst1[3] = amd_bytealign_S (src_r1[1], src_r1[0], 3);
-      dst1[2] = amd_bytealign_S (src_r1[0], src_r0[3], 3);
-      dst1[1] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
-      dst1[0] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
-      dst0[3] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
-      dst0[2] = src_l0[2] | src_r0[0] <<  8;
-      break;
-    case 8:
-      dst1[3] = src_r1[1];
-      dst1[2] = src_r1[0];
-      dst1[1] = src_r0[3];
-      dst1[0] = src_r0[2];
-      dst0[3] = src_r0[1];
-      dst0[2] = src_r0[0];
-      break;
-    case 7:
-      dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 1);
-      dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 1);
-      dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 1);
-      dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
-      dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
-      dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
-      dst0[1] = src_l0[1] | src_r0[0] << 24;
-      break;
-    case 6:
-      dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 2);
-      dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 2);
-      dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 2);
-      dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
-      dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
-      dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
-      dst0[1] = src_l0[1] | src_r0[0] << 16;
-      break;
-    case 5:
-      dst1[3] = amd_bytealign_S (src_r1[2], src_r1[1], 3);
-      dst1[2] = amd_bytealign_S (src_r1[1], src_r1[0], 3);
-      dst1[1] = amd_bytealign_S (src_r1[0], src_r0[3], 3);
-      dst1[0] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
-      dst0[3] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
-      dst0[2] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
-      dst0[1] = src_l0[1] | src_r0[0] <<  8;
-      break;
-    case 4:
-      dst1[3] = src_r1[2];
-      dst1[2] = src_r1[1];
-      dst1[1] = src_r1[0];
-      dst1[0] = src_r0[3];
-      dst0[3] = src_r0[2];
-      dst0[2] = src_r0[1];
-      dst0[1] = src_r0[0];
-      break;
-    case 3:
-      dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 1);
-      dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 1);
-      dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 1);
-      dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 1);
-      dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 1);
-      dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 1);
-      dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 1);
-      dst0[0] = src_l0[0] | src_r0[0] << 24;
-      break;
-    case 2:
-      dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 2);
-      dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 2);
-      dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 2);
-      dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 2);
-      dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 2);
-      dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 2);
-      dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 2);
-      dst0[0] = src_l0[0] | src_r0[0] << 16;
-      break;
-    case 1:
-      dst1[3] = amd_bytealign_S (src_r1[3], src_r1[2], 3);
-      dst1[2] = amd_bytealign_S (src_r1[2], src_r1[1], 3);
-      dst1[1] = amd_bytealign_S (src_r1[1], src_r1[0], 3);
-      dst1[0] = amd_bytealign_S (src_r1[0], src_r0[3], 3);
-      dst0[3] = amd_bytealign_S (src_r0[3], src_r0[2], 3);
-      dst0[2] = amd_bytealign_S (src_r0[2], src_r0[1], 3);
-      dst0[1] = amd_bytealign_S (src_r0[1], src_r0[0], 3);
-      dst0[0] = src_l0[0] | src_r0[0] <<  8;
-      break;
-    case 0:
-      dst1[3] = src_r1[3];
-      dst1[2] = src_r1[2];
-      dst1[1] = src_r1[1];
-      dst1[0] = src_r1[0];
-      dst0[3] = src_r0[3];
-      dst0[2] = src_r0[2];
-      dst0[1] = src_r0[1];
-      dst0[0] = src_r0[0];
-      break;
-  }
-//  #endif
-}
-
-inline void reverse_block (u32 in0[4], u32 in1[4], u32 out0[4], u32 out1[4], const u32 len)
-{
-  rshift_block_N (in0, in1, out0, out1, 32 - len);
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  tib40[0] = out1[3];
-  tib40[1] = out1[2];
-  tib40[2] = out1[1];
-  tib40[3] = out1[0];
-  tib41[0] = out0[3];
-  tib41[1] = out0[2];
-  tib41[2] = out0[1];
-  tib41[3] = out0[0];
-
-  out0[0] = swap32_S (tib40[0]);
-  out0[1] = swap32_S (tib40[1]);
-  out0[2] = swap32_S (tib40[2]);
-  out0[3] = swap32_S (tib40[3]);
-  out1[0] = swap32_S (tib41[0]);
-  out1[1] = swap32_S (tib41[1]);
-  out1[2] = swap32_S (tib41[2]);
-  out1[3] = swap32_S (tib41[3]);
-}
-
-inline u32 rule_op_mangle_lrest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  buf0[0] |= (generate_cmask (buf0[0]));
-  buf0[1] |= (generate_cmask (buf0[1]));
-  buf0[2] |= (generate_cmask (buf0[2]));
-  buf0[3] |= (generate_cmask (buf0[3]));
-  buf1[0] |= (generate_cmask (buf1[0]));
-  buf1[1] |= (generate_cmask (buf1[1]));
-  buf1[2] |= (generate_cmask (buf1[2]));
-  buf1[3] |= (generate_cmask (buf1[3]));
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_urest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  buf0[0] &= ~(generate_cmask (buf0[0]));
-  buf0[1] &= ~(generate_cmask (buf0[1]));
-  buf0[2] &= ~(generate_cmask (buf0[2]));
-  buf0[3] &= ~(generate_cmask (buf0[3]));
-  buf1[0] &= ~(generate_cmask (buf1[0]));
-  buf1[1] &= ~(generate_cmask (buf1[1]));
-  buf1[2] &= ~(generate_cmask (buf1[2]));
-  buf1[3] &= ~(generate_cmask (buf1[3]));
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_lrest_ufirst (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  rule_op_mangle_lrest (p0, p1, buf0, buf1, in_len);
-
-  buf0[0] &= ~(0x00000020 & generate_cmask (buf0[0]));
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_urest_lfirst (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  rule_op_mangle_urest (p0, p1, buf0, buf1, in_len);
-
-  buf0[0] |= (0x00000020 & generate_cmask (buf0[0]));
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_trest (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  buf0[0] ^= (generate_cmask (buf0[0]));
-  buf0[1] ^= (generate_cmask (buf0[1]));
-  buf0[2] ^= (generate_cmask (buf0[2]));
-  buf0[3] ^= (generate_cmask (buf0[3]));
-  buf1[0] ^= (generate_cmask (buf1[0]));
-  buf1[1] ^= (generate_cmask (buf1[1]));
-  buf1[2] ^= (generate_cmask (buf1[2]));
-  buf1[3] ^= (generate_cmask (buf1[3]));
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_toggle_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  const u32 tmp = 0x20u << ((p0 & 3) * 8);
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] ^= (tmp & generate_cmask (buf0[0])); break;
-    case  1:  buf0[1] ^= (tmp & generate_cmask (buf0[1])); break;
-    case  2:  buf0[2] ^= (tmp & generate_cmask (buf0[2])); break;
-    case  3:  buf0[3] ^= (tmp & generate_cmask (buf0[3])); break;
-    case  4:  buf1[0] ^= (tmp & generate_cmask (buf1[0])); break;
-    case  5:  buf1[1] ^= (tmp & generate_cmask (buf1[1])); break;
-    case  6:  buf1[2] ^= (tmp & generate_cmask (buf1[2])); break;
-    case  7:  buf1[3] ^= (tmp & generate_cmask (buf1[3])); break;
-  }
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_reverse (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  reverse_block (buf0, buf1, buf0, buf1, in_len);
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_dupeword (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if ((in_len + in_len) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  append_block8 (out_len, buf0, buf1, buf0, buf1, buf0, buf1);
-
-  out_len += in_len;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_dupeword_times (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (((in_len * p0) + in_len) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  tib40[0] = buf0[0];
-  tib40[1] = buf0[1];
-  tib40[2] = buf0[2];
-  tib40[3] = buf0[3];
-  tib41[0] = buf1[0];
-  tib41[1] = buf1[1];
-  tib41[2] = buf1[2];
-  tib41[3] = buf1[3];
-
-  for (u32 i = 0; i < p0; i++)
-  {
-    append_block8 (out_len, buf0, buf1, buf0, buf1, tib40, tib41);
-
-    out_len += in_len;
-  }
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_reflect (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if ((in_len + in_len) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  reverse_block (buf0, buf1, tib40, tib41, out_len);
-
-  append_block8 (out_len, buf0, buf1, buf0, buf1, tib40, tib41);
-
-  out_len += in_len;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_append (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if ((in_len + 1) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  append_block1 (out_len, buf0, buf1, p0);
-
-  out_len++;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_prepend (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if ((in_len + 1) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  rshift_block (buf0, buf1, buf0, buf1);
-
-  buf0[0] = buf0[0] | p0;
-
-  out_len++;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_rotate_left (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (in_len == 0) return (in_len);
-
-  const u32 in_len1 = in_len - 1;
-
-  const u32 sh = (in_len1 & 3) * 8;
-
-  const u32 tmp = (buf0[0] & 0xff) << sh;
-
-  lshift_block (buf0, buf1, buf0, buf1);
-
-  switch (in_len1 / 4)
-  {
-    case  0:  buf0[0] |= tmp; break;
-    case  1:  buf0[1] |= tmp; break;
-    case  2:  buf0[2] |= tmp; break;
-    case  3:  buf0[3] |= tmp; break;
-    case  4:  buf1[0] |= tmp; break;
-    case  5:  buf1[1] |= tmp; break;
-    case  6:  buf1[2] |= tmp; break;
-    case  7:  buf1[3] |= tmp; break;
-  }
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_rotate_right (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (in_len == 0) return (in_len);
-
-  const u32 in_len1 = in_len - 1;
-
-  const u32 sh = (in_len1 & 3) * 8;
-
-  u32 tmp = 0;
-
-  switch (in_len1 / 4)
-  {
-    case  0:  tmp = (buf0[0] >> sh) & 0xff; break;
-    case  1:  tmp = (buf0[1] >> sh) & 0xff; break;
-    case  2:  tmp = (buf0[2] >> sh) & 0xff; break;
-    case  3:  tmp = (buf0[3] >> sh) & 0xff; break;
-    case  4:  tmp = (buf1[0] >> sh) & 0xff; break;
-    case  5:  tmp = (buf1[1] >> sh) & 0xff; break;
-    case  6:  tmp = (buf1[2] >> sh) & 0xff; break;
-    case  7:  tmp = (buf1[3] >> sh) & 0xff; break;
-  }
-
-  rshift_block (buf0, buf1, buf0, buf1);
-
-  buf0[0] |= tmp;
-
-  truncate_right (buf0, buf1, in_len);
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_delete_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (in_len == 0) return (in_len);
-
-  const u32 in_len1 = in_len - 1;
-
-  lshift_block (buf0, buf1, buf0, buf1);
-
-  return in_len1;
-}
-
-inline u32 rule_op_mangle_delete_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (in_len == 0) return (in_len);
-
-  const u32 in_len1 = in_len - 1;
-
-  const u32 tmp = (1 << ((in_len1 & 3) * 8)) - 1;
-
-  switch (in_len1 / 4)
-  {
-    case  0:  buf0[0] &= tmp; break;
-    case  1:  buf0[1] &= tmp; break;
-    case  2:  buf0[2] &= tmp; break;
-    case  3:  buf0[3] &= tmp; break;
-    case  4:  buf1[0] &= tmp; break;
-    case  5:  buf1[1] &= tmp; break;
-    case  6:  buf1[2] &= tmp; break;
-    case  7:  buf1[3] &= tmp; break;
-  }
-
-  return in_len1;
-}
-
-inline u32 rule_op_mangle_delete_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  u32 out_len = in_len;
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  lshift_block (buf0, buf1, tib40, tib41);
-
-  const u32 ml = (1 << ((p0 & 3) * 8)) - 1;
-  const u32 mr = ~ml;
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] =  (buf0[0] & ml)
-                      | (tib40[0] & mr);
-              buf0[1] =  tib40[1];
-              buf0[2] =  tib40[2];
-              buf0[3] =  tib40[3];
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  1:  buf0[1] =  (buf0[1] & ml)
-                      | (tib40[1] & mr);
-              buf0[2] =  tib40[2];
-              buf0[3] =  tib40[3];
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  2:  buf0[2] =  (buf0[2] & ml)
-                      | (tib40[2] & mr);
-              buf0[3] =  tib40[3];
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  3:  buf0[3] =  (buf0[3] & ml)
-                      | (tib40[3] & mr);
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  4:  buf1[0] =  (buf1[0] & ml)
-                      | (tib41[0] & mr);
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  5:  buf1[1] =  (buf1[1] & ml)
-                      | (tib41[1] & mr);
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  6:  buf1[2] =  (buf1[2] & ml)
-                      | (tib41[2] & mr);
-              buf1[3] =  tib41[3];
-              break;
-    case  7:  buf1[3] =  (buf1[3] & ml)
-                      | (tib41[3] & mr);
-              break;
-  }
-
-  out_len--;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_extract (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  if ((p0 + p1) > in_len) return (in_len);
-
-  u32 out_len = p1;
-
-  lshift_block_N (buf0, buf1, buf0, buf1, p0);
-
-  truncate_right (buf0, buf1, out_len);
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_omit (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  if ((p0 + p1) > in_len) return (in_len);
-
-  u32 out_len = in_len;
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  tib40[0] = 0;
-  tib40[1] = 0;
-  tib40[2] = 0;
-  tib40[3] = 0;
-  tib41[0] = 0;
-  tib41[1] = 0;
-  tib41[2] = 0;
-  tib41[3] = 0;
-
-  lshift_block_N (buf0, buf1, tib40, tib41, p1);
-
-  const u32 ml = (1 << ((p0 & 3) * 8)) - 1;
-  const u32 mr = ~ml;
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] =  (buf0[0] & ml)
-                      | (tib40[0] & mr);
-              buf0[1] =  tib40[1];
-              buf0[2] =  tib40[2];
-              buf0[3] =  tib40[3];
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  1:  buf0[1] =  (buf0[1] & ml)
-                      | (tib40[1] & mr);
-              buf0[2] =  tib40[2];
-              buf0[3] =  tib40[3];
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  2:  buf0[2] =  (buf0[2] & ml)
-                      | (tib40[2] & mr);
-              buf0[3] =  tib40[3];
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  3:  buf0[3] =  (buf0[3] & ml)
-                      | (tib40[3] & mr);
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  4:  buf1[0] =  (buf1[0] & ml)
-                      | (tib41[0] & mr);
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  5:  buf1[1] =  (buf1[1] & ml)
-                      | (tib41[1] & mr);
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  6:  buf1[2] =  (buf1[2] & ml)
-                      | (tib41[2] & mr);
-              buf1[3] =  tib41[3];
-              break;
-    case  7:  buf1[3] =  (buf1[3] & ml)
-                      | (tib41[3] & mr);
-              break;
-  }
-
-  out_len -= p1;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_insert (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 > in_len) return (in_len);
-
-  if ((in_len + 1) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  rshift_block (buf0, buf1, tib40, tib41);
-
-  const u32 p1n = p1 << ((p0 & 3) * 8);
-
-  const u32 ml = (1 << ((p0 & 3) * 8)) - 1;
-
-  const u32 mr = 0xffffff00 << ((p0 & 3) * 8);
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] =  (buf0[0] & ml) | p1n | (tib40[0] & mr);
-              buf0[1] =  tib40[1];
-              buf0[2] =  tib40[2];
-              buf0[3] =  tib40[3];
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  1:  buf0[1] =  (buf0[1] & ml) | p1n | (tib40[1] & mr);
-              buf0[2] =  tib40[2];
-              buf0[3] =  tib40[3];
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  2:  buf0[2] =  (buf0[2] & ml) | p1n | (tib40[2] & mr);
-              buf0[3] =  tib40[3];
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  3:  buf0[3] =  (buf0[3] & ml) | p1n | (tib40[3] & mr);
-              buf1[0] =  tib41[0];
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  4:  buf1[0] =  (buf1[0] & ml) | p1n | (tib41[0] & mr);
-              buf1[1] =  tib41[1];
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  5:  buf1[1] =  (buf1[1] & ml) | p1n | (tib41[1] & mr);
-              buf1[2] =  tib41[2];
-              buf1[3] =  tib41[3];
-              break;
-    case  6:  buf1[2] =  (buf1[2] & ml) | p1n | (tib41[2] & mr);
-              buf1[3] =  tib41[3];
-              break;
-    case  7:  buf1[3] =  (buf1[3] & ml) | p1n | (tib41[3] & mr);
-              break;
-  }
-
-  out_len++;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_overstrike (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  const u32 p1n = p1 << ((p0 & 3) * 8);
-
-  const u32 m = ~(0xffu << ((p0 & 3) * 8));
-
-  switch (p0 / 4)
-  {
-    case  0: buf0[0] = (buf0[0] & m) | p1n; break;
-    case  1: buf0[1] = (buf0[1] & m) | p1n; break;
-    case  2: buf0[2] = (buf0[2] & m) | p1n; break;
-    case  3: buf0[3] = (buf0[3] & m) | p1n; break;
-    case  4: buf1[0] = (buf1[0] & m) | p1n; break;
-    case  5: buf1[1] = (buf1[1] & m) | p1n; break;
-    case  6: buf1[2] = (buf1[2] & m) | p1n; break;
-    case  7: buf1[3] = (buf1[3] & m) | p1n; break;
-  }
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_truncate_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  truncate_right (buf0, buf1, p0);
-
-  return p0;
-}
-
-inline u32 rule_op_mangle_replace (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  #ifdef IS_NV
-  for (u32 i = 0; i < in_len; i++)
-  {
-    switch (i)
-    {
-      case  0:  if ((__byte_perm_S (buf0[0], 0, 0x6540)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7650);
-                break;
-      case  1:  if ((__byte_perm_S (buf0[0], 0, 0x6541)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7604);
-                break;
-      case  2:  if ((__byte_perm_S (buf0[0], 0, 0x6542)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x7054);
-                break;
-      case  3:  if ((__byte_perm_S (buf0[0], 0, 0x6543)) == p0) buf0[0] = __byte_perm_S (p1, buf0[0], 0x0654);
-                break;
-      case  4:  if ((__byte_perm_S (buf0[1], 0, 0x6540)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7650);
-                break;
-      case  5:  if ((__byte_perm_S (buf0[1], 0, 0x6541)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7604);
-                break;
-      case  6:  if ((__byte_perm_S (buf0[1], 0, 0x6542)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x7054);
-                break;
-      case  7:  if ((__byte_perm_S (buf0[1], 0, 0x6543)) == p0) buf0[1] = __byte_perm_S (p1, buf0[1], 0x0654);
-                break;
-      case  8:  if ((__byte_perm_S (buf0[2], 0, 0x6540)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7650);
-                break;
-      case  9:  if ((__byte_perm_S (buf0[2], 0, 0x6541)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7604);
-                break;
-      case 10:  if ((__byte_perm_S (buf0[2], 0, 0x6542)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x7054);
-                break;
-      case 11:  if ((__byte_perm_S (buf0[2], 0, 0x6543)) == p0) buf0[2] = __byte_perm_S (p1, buf0[2], 0x0654);
-                break;
-      case 12:  if ((__byte_perm_S (buf0[3], 0, 0x6540)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7650);
-                break;
-      case 13:  if ((__byte_perm_S (buf0[3], 0, 0x6541)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7604);
-                break;
-      case 14:  if ((__byte_perm_S (buf0[3], 0, 0x6542)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x7054);
-                break;
-      case 15:  if ((__byte_perm_S (buf0[3], 0, 0x6543)) == p0) buf0[3] = __byte_perm_S (p1, buf0[3], 0x0654);
-                break;
-      case 16:  if ((__byte_perm_S (buf1[0], 0, 0x6540)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7650);
-                break;
-      case 17:  if ((__byte_perm_S (buf1[0], 0, 0x6541)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7604);
-                break;
-      case 18:  if ((__byte_perm_S (buf1[0], 0, 0x6542)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x7054);
-                break;
-      case 19:  if ((__byte_perm_S (buf1[0], 0, 0x6543)) == p0) buf1[0] = __byte_perm_S (p1, buf1[0], 0x0654);
-                break;
-      case 20:  if ((__byte_perm_S (buf1[1], 0, 0x6540)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7650);
-                break;
-      case 21:  if ((__byte_perm_S (buf1[1], 0, 0x6541)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7604);
-                break;
-      case 22:  if ((__byte_perm_S (buf1[1], 0, 0x6542)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x7054);
-                break;
-      case 23:  if ((__byte_perm_S (buf1[1], 0, 0x6543)) == p0) buf1[1] = __byte_perm_S (p1, buf1[1], 0x0654);
-                break;
-      case 24:  if ((__byte_perm_S (buf1[2], 0, 0x6540)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7650);
-                break;
-      case 25:  if ((__byte_perm_S (buf1[2], 0, 0x6541)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7604);
-                break;
-      case 26:  if ((__byte_perm_S (buf1[2], 0, 0x6542)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x7054);
-                break;
-      case 27:  if ((__byte_perm_S (buf1[2], 0, 0x6543)) == p0) buf1[2] = __byte_perm_S (p1, buf1[2], 0x0654);
-                break;
-      case 28:  if ((__byte_perm_S (buf1[3], 0, 0x6540)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7650);
-                break;
-      case 29:  if ((__byte_perm_S (buf1[3], 0, 0x6541)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7604);
-                break;
-      case 30:  if ((__byte_perm_S (buf1[3], 0, 0x6542)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x7054);
-                break;
-      case 31:  if ((__byte_perm_S (buf1[3], 0, 0x6543)) == p0) buf1[3] = __byte_perm_S (p1, buf1[3], 0x0654);
-                break;
-    }
-  }
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  const uchar4 tmp0 = (uchar4) (p0);
-  const uchar4 tmp1 = (uchar4) (p1);
-
-  uchar4 tmp;
-
-  tmp = as_uchar4 (buf0[0]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[0] = as_uint (tmp);
-  tmp = as_uchar4 (buf0[1]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[1] = as_uint (tmp);
-  tmp = as_uchar4 (buf0[2]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[2] = as_uint (tmp);
-  tmp = as_uchar4 (buf0[3]); tmp = select (tmp, tmp1, tmp == tmp0); buf0[3] = as_uint (tmp);
-  tmp = as_uchar4 (buf1[0]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[0] = as_uint (tmp);
-  tmp = as_uchar4 (buf1[1]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[1] = as_uint (tmp);
-  tmp = as_uchar4 (buf1[2]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[2] = as_uint (tmp);
-  tmp = as_uchar4 (buf1[3]); tmp = select (tmp, tmp1, tmp == tmp0); buf1[3] = as_uint (tmp);
-  #endif
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_purgechar (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  // TODO
-  return in_len;
-}
-
-inline u32 rule_op_mangle_togglecase_rec (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  // TODO
-  return in_len;
-}
-
-inline u32 rule_op_mangle_dupechar_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if ( in_len       ==  0) return (in_len);
-  if ((in_len + p0) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  const u32 tmp = buf0[0] & 0xFF;
-
-  rshift_block_N (buf0, buf1, buf0, buf1, p0);
-
-  #ifdef IS_NV
-  switch (p0)
-  {
-    case  1:  buf0[0] |= tmp;
-              break;
-    case  2:  buf0[0] |= __byte_perm_S (tmp, 0, 0x5400);
-              break;
-    case  3:  buf0[0] |= __byte_perm_S (tmp, 0, 0x4000);
-              break;
-    case  4:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              break;
-    case  5:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= tmp;
-              break;
-    case  6:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x5400);
-              break;
-    case  7:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x4000);
-              break;
-    case  8:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              break;
-    case  9:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= tmp;
-              break;
-    case 10:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x5400);
-              break;
-    case 11:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x4000);
-              break;
-    case 12:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              break;
-    case 13:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= tmp;
-              break;
-    case 14:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x5400);
-              break;
-    case 15:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x4000);
-              break;
-    case 16:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              break;
-    case 17:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= tmp;
-              break;
-    case 18:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x5400);
-              break;
-    case 19:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x4000);
-              break;
-    case 20:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              break;
-    case 21:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= tmp;
-              break;
-    case 22:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x5400);
-              break;
-    case 23:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x4000);
-              break;
-    case 24:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              break;
-    case 25:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[2] |= tmp;
-              break;
-    case 26:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[2] |= __byte_perm_S (tmp, 0, 0x5400);
-              break;
-    case 27:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[2] |= __byte_perm_S (tmp, 0, 0x4000);
-              break;
-    case 28:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              break;
-    case 29:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[3] |= tmp;
-              break;
-    case 30:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[3] |= __byte_perm_S (tmp, 0, 0x5400);
-              break;
-    case 31:  buf0[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf0[3] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[0] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[1] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[2] |= __byte_perm_S (tmp, 0, 0x0000);
-              buf1[3] |= __byte_perm_S (tmp, 0, 0x4000);
-              break;
-  }
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  switch (p0)
-  {
-    case  1:  buf0[0] |= tmp <<  0;
-              break;
-    case  2:  buf0[0] |= tmp <<  0 | tmp << 8;
-              break;
-    case  3:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16;
-              break;
-    case  4:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              break;
-    case  5:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0;
-              break;
-    case  6:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8;
-              break;
-    case  7:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16;
-              break;
-    case  8:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              break;
-    case  9:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0;
-              break;
-    case 10:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8;
-              break;
-    case 11:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16;
-              break;
-    case 12:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              break;
-    case 13:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0;
-              break;
-    case 14:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8;
-              break;
-    case 15:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16;
-              break;
-    case 16:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              break;
-    case 17:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0;
-              break;
-    case 18:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8;
-              break;
-    case 19:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16;
-              break;
-    case 20:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              break;
-    case 21:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0;
-              break;
-    case 22:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8;
-              break;
-    case 23:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16;
-              break;
-    case 24:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              break;
-    case 25:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[2] |= tmp <<  0;
-              break;
-    case 26:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[2] |= tmp <<  0 | tmp << 8;
-              break;
-    case 27:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16;
-              break;
-    case 28:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              break;
-    case 29:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[3] |= tmp <<  0;
-              break;
-    case 30:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[3] |= tmp <<  0 | tmp << 8;
-              break;
-    case 31:  buf0[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf0[3] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[0] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[1] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[2] |= tmp <<  0 | tmp << 8 | tmp << 16 | tmp << 24;
-              buf1[3] |= tmp <<  0 | tmp << 8 | tmp << 16;
-              break;
-  }
-  #endif
-
-  out_len += p0;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_dupechar_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if ( in_len       ==  0) return (in_len);
-  if ((in_len + p0) >= 32) return (in_len);
-
-  const u32 in_len1 = in_len - 1;
-
-  const u32 sh = (in_len1 & 3) * 8;
-
-  u32 tmp = 0;
-
-  switch (in_len1 / 4)
-  {
-    case  0:  tmp = (buf0[0] >> sh) & 0xff; break;
-    case  1:  tmp = (buf0[1] >> sh) & 0xff; break;
-    case  2:  tmp = (buf0[2] >> sh) & 0xff; break;
-    case  3:  tmp = (buf0[3] >> sh) & 0xff; break;
-    case  4:  tmp = (buf1[0] >> sh) & 0xff; break;
-    case  5:  tmp = (buf1[1] >> sh) & 0xff; break;
-    case  6:  tmp = (buf1[2] >> sh) & 0xff; break;
-    case  7:  tmp = (buf1[3] >> sh) & 0xff; break;
-  }
-
-  u32 out_len = in_len;
-
-  for (u32 i = 0; i < p0; i++)
-  {
-    append_block1 (out_len, buf0, buf1, tmp);
-
-    out_len++;
-  }
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_dupechar_all (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if ( in_len           ==  0) return (in_len);
-  if ((in_len + in_len) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  #ifdef IS_NV
-  tib40[0] = __byte_perm_S (buf0[0], 0, 0x1100);
-  tib40[1] = __byte_perm_S (buf0[0], 0, 0x3322);
-  tib40[2] = __byte_perm_S (buf0[1], 0, 0x1100);
-  tib40[3] = __byte_perm_S (buf0[1], 0, 0x3322);
-  tib41[0] = __byte_perm_S (buf0[2], 0, 0x1100);
-  tib41[1] = __byte_perm_S (buf0[2], 0, 0x3322);
-  tib41[2] = __byte_perm_S (buf0[3], 0, 0x1100);
-  tib41[3] = __byte_perm_S (buf0[3], 0, 0x3322);
-
-  buf0[0] = tib40[0];
-  buf0[1] = tib40[1];
-  buf0[2] = tib40[2];
-  buf0[3] = tib40[3];
-  buf1[0] = tib41[0];
-  buf1[1] = tib41[1];
-  buf1[2] = tib41[2];
-  buf1[3] = tib41[3];
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  tib40[0] = ((buf0[0] & 0x000000FF) <<  0) | ((buf0[0] & 0x0000FF00) <<  8);
-  tib40[1] = ((buf0[0] & 0x00FF0000) >> 16) | ((buf0[0] & 0xFF000000) >>  8);
-  tib40[2] = ((buf0[1] & 0x000000FF) <<  0) | ((buf0[1] & 0x0000FF00) <<  8);
-  tib40[3] = ((buf0[1] & 0x00FF0000) >> 16) | ((buf0[1] & 0xFF000000) >>  8);
-  tib41[0] = ((buf0[2] & 0x000000FF) <<  0) | ((buf0[2] & 0x0000FF00) <<  8);
-  tib41[1] = ((buf0[2] & 0x00FF0000) >> 16) | ((buf0[2] & 0xFF000000) >>  8);
-  tib41[2] = ((buf0[3] & 0x000000FF) <<  0) | ((buf0[3] & 0x0000FF00) <<  8);
-  tib41[3] = ((buf0[3] & 0x00FF0000) >> 16) | ((buf0[3] & 0xFF000000) >>  8);
-
-  buf0[0] = tib40[0] | (tib40[0] <<  8);
-  buf0[1] = tib40[1] | (tib40[1] <<  8);
-  buf0[2] = tib40[2] | (tib40[2] <<  8);
-  buf0[3] = tib40[3] | (tib40[3] <<  8);
-  buf1[0] = tib41[0] | (tib41[0] <<  8);
-  buf1[1] = tib41[1] | (tib41[1] <<  8);
-  buf1[2] = tib41[2] | (tib41[2] <<  8);
-  buf1[3] = tib41[3] | (tib41[3] <<  8);
-  #endif
-
-  out_len = out_len + out_len;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_switch_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (in_len < 2) return (in_len);
-
-  #ifdef IS_NV
-  buf0[0] = __byte_perm_S (buf0[0], 0, 0x3201);
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  buf0[0] = (buf0[0] & 0xFFFF0000) | ((buf0[0] << 8) & 0x0000FF00) | ((buf0[0] >> 8) & 0x000000FF);
-  #endif
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_switch_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (in_len < 2) return (in_len);
-
-  #ifdef IS_NV
-  switch (in_len)
-  {
-    case  2:  buf0[0] = __byte_perm_S (buf0[0],       0, 0x5401);
-              break;
-    case  3:  buf0[0] = __byte_perm_S (buf0[0],       0, 0x4120);
-              break;
-    case  4:  buf0[0] = __byte_perm_S (buf0[0],       0, 0x2310);
-              break;
-    case  5:  buf0[1] = __byte_perm_S (buf0[1], buf0[0], 0x7210);
-              buf0[0] = __byte_perm_S (buf0[0], buf0[1], 0x4210);
-              buf0[1] = __byte_perm_S (buf0[1],       0, 0x6543);
-              break;
-    case  6:  buf0[1] = __byte_perm_S (buf0[1],       0, 0x5401);
-              break;
-    case  7:  buf0[1] = __byte_perm_S (buf0[1],       0, 0x4120);
-              break;
-    case  8:  buf0[1] = __byte_perm_S (buf0[1],       0, 0x2310);
-              break;
-    case  9:  buf0[2] = __byte_perm_S (buf0[2], buf0[1], 0x7210);
-              buf0[1] = __byte_perm_S (buf0[1], buf0[2], 0x4210);
-              buf0[2] = __byte_perm_S (buf0[2],       0, 0x6543);
-              break;
-    case 10:  buf0[2] = __byte_perm_S (buf0[2],       0, 0x5401);
-              break;
-    case 11:  buf0[2] = __byte_perm_S (buf0[2],       0, 0x4120);
-              break;
-    case 12:  buf0[2] = __byte_perm_S (buf0[2],       0, 0x2310);
-              break;
-    case 13:  buf0[3] = __byte_perm_S (buf0[3], buf0[2], 0x7210);
-              buf0[2] = __byte_perm_S (buf0[2], buf0[3], 0x4210);
-              buf0[3] = __byte_perm_S (buf0[3],       0, 0x6543);
-              break;
-    case 14:  buf0[3] = __byte_perm_S (buf0[3],       0, 0x5401);
-              break;
-    case 15:  buf0[3] = __byte_perm_S (buf0[3],       0, 0x4120);
-              break;
-    case 16:  buf0[3] = __byte_perm_S (buf0[3],       0, 0x2310);
-              break;
-    case 17:  buf1[0] = __byte_perm_S (buf1[0], buf0[3], 0x7210);
-              buf0[3] = __byte_perm_S (buf0[3], buf1[0], 0x4210);
-              buf1[0] = __byte_perm_S (buf1[0],       0, 0x6543);
-              break;
-    case 18:  buf1[0] = __byte_perm_S (buf1[0],       0, 0x5401);
-              break;
-    case 19:  buf1[0] = __byte_perm_S (buf1[0],       0, 0x4120);
-              break;
-    case 20:  buf1[0] = __byte_perm_S (buf1[0],       0, 0x2310);
-              break;
-    case 21:  buf1[1] = __byte_perm_S (buf1[1], buf1[0], 0x7210);
-              buf1[0] = __byte_perm_S (buf1[0], buf1[1], 0x4210);
-              buf1[1] = __byte_perm_S (buf1[1],       0, 0x6543);
-              break;
-    case 22:  buf1[1] = __byte_perm_S (buf1[1],       0, 0x5401);
-              break;
-    case 23:  buf1[1] = __byte_perm_S (buf1[1],       0, 0x4120);
-              break;
-    case 24:  buf1[1] = __byte_perm_S (buf1[1],       0, 0x2310);
-              break;
-    case 25:  buf1[2] = __byte_perm_S (buf1[2], buf1[1], 0x7210);
-              buf1[1] = __byte_perm_S (buf1[1], buf1[2], 0x4210);
-              buf1[2] = __byte_perm_S (buf1[2],       0, 0x6543);
-              break;
-    case 26:  buf1[2] = __byte_perm_S (buf1[2],       0, 0x5401);
-              break;
-    case 27:  buf1[2] = __byte_perm_S (buf1[2],       0, 0x4120);
-              break;
-    case 28:  buf1[2] = __byte_perm_S (buf1[2],       0, 0x2310);
-              break;
-    case 29:  buf1[3] = __byte_perm_S (buf1[3], buf1[2], 0x7210);
-              buf1[2] = __byte_perm_S (buf1[2], buf1[3], 0x4210);
-              buf1[3] = __byte_perm_S (buf1[3],       0, 0x6543);
-              break;
-    case 30:  buf1[3] = __byte_perm_S (buf1[3],       0, 0x5401);
-              break;
-    case 31:  buf1[3] = __byte_perm_S (buf1[3],       0, 0x4120);
-              break;
-  }
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  switch (in_len)
-  {
-    case  2:  buf0[0] = ((buf0[0] << 8) & 0x0000FF00) | ((buf0[0] >> 8) & 0x000000FF);
-              break;
-    case  3:  buf0[0] =  (buf0[0] & 0x000000FF) | ((buf0[0] <<  8) & 0x00FF0000) | ((buf0[0] >>  8) & 0x0000FF00);
-              break;
-    case  4:  buf0[0] =  (buf0[0] & 0x0000FFFF) | ((buf0[0] <<  8) & 0xFF000000) | ((buf0[0] >>  8) & 0x00FF0000);
-              break;
-    case  5:  buf0[1] =  (buf0[0] & 0xFF000000) |   buf0[1];
-              buf0[0] =  (buf0[0] & 0x00FFFFFF) |  (buf0[1] << 24);
-              buf0[1] =  (buf0[1] >> 24);
-              break;
-    case  6:  buf0[1] = ((buf0[1] << 8) & 0x0000FF00) | ((buf0[1] >> 8) & 0x000000FF);
-              break;
-    case  7:  buf0[1] =  (buf0[1] & 0x000000FF) | ((buf0[1] <<  8) & 0x00FF0000) | ((buf0[1] >>  8) & 0x0000FF00);
-              break;
-    case  8:  buf0[1] =  (buf0[1] & 0x0000FFFF) | ((buf0[1] <<  8) & 0xFF000000) | ((buf0[1] >>  8) & 0x00FF0000);
-              break;
-    case  9:  buf0[2] =  (buf0[1] & 0xFF000000) |   buf0[2];
-              buf0[1] =  (buf0[1] & 0x00FFFFFF) |  (buf0[2] << 24);
-              buf0[2] =  (buf0[2] >> 24);
-              break;
-    case 10:  buf0[2] = ((buf0[2] << 8) & 0x0000FF00) | ((buf0[2] >> 8) & 0x000000FF);
-              break;
-    case 11:  buf0[2] =  (buf0[2] & 0x000000FF) | ((buf0[2] <<  8) & 0x00FF0000) | ((buf0[2] >>  8) & 0x0000FF00);
-              break;
-    case 12:  buf0[2] =  (buf0[2] & 0x0000FFFF) | ((buf0[2] <<  8) & 0xFF000000) | ((buf0[2] >>  8) & 0x00FF0000);
-              break;
-    case 13:  buf0[3] =  (buf0[2] & 0xFF000000) |   buf0[3];
-              buf0[2] =  (buf0[2] & 0x00FFFFFF) |  (buf0[3] << 24);
-              buf0[3] =  (buf0[3] >> 24);
-              break;
-    case 14:  buf0[3] = ((buf0[3] << 8) & 0x0000FF00) | ((buf0[3] >> 8) & 0x000000FF);
-              break;
-    case 15:  buf0[3] =  (buf0[3] & 0x000000FF) |  ((buf0[3] <<  8) & 0x00FF0000) | ((buf0[3] >>  8) & 0x0000FF00);
-              break;
-    case 16:  buf0[3] =  (buf0[3] & 0x0000FFFF) | ((buf0[3] <<  8) & 0xFF000000) | ((buf0[3] >>  8) & 0x00FF0000);
-              break;
-    case 17:  buf1[0] =  (buf0[3] & 0xFF000000) |   buf1[0];
-              buf0[3] =  (buf0[3] & 0x00FFFFFF) |  (buf1[0] << 24);
-              buf1[0] =  (buf1[0] >> 24);
-              break;
-    case 18:  buf1[0] = ((buf1[0] << 8) & 0x0000FF00) | ((buf1[0] >> 8) & 0x000000FF);
-              break;
-    case 19:  buf1[0] =  (buf1[0] & 0x000000FF) | ((buf1[0] <<  8) & 0x00FF0000) | ((buf1[0] >>  8) & 0x0000FF00);
-              break;
-    case 20:  buf1[0] =  (buf1[0] & 0x0000FFFF) | ((buf1[0] <<  8) & 0xFF000000) | ((buf1[0] >>  8) & 0x00FF0000);
-              break;
-    case 21:  buf1[1] =  (buf1[0] & 0xFF000000) |   buf1[1];
-              buf1[0] =  (buf1[0] & 0x00FFFFFF) |  (buf1[1] << 24);
-              buf1[1] =  (buf1[1] >> 24);
-              break;
-    case 22:  buf1[1] = ((buf1[1] << 8) & 0x0000FF00) | ((buf1[1] >> 8) & 0x000000FF);
-              break;
-    case 23:  buf1[1] =  (buf1[1] & 0x000000FF) | ((buf1[1] <<  8) & 0x00FF0000) | ((buf1[1] >>  8) & 0x0000FF00);
-              break;
-    case 24:  buf1[1] =  (buf1[1] & 0x0000FFFF) | ((buf1[1] <<  8) & 0xFF000000) | ((buf1[1] >>  8) & 0x00FF0000);
-              break;
-    case 25:  buf1[2] =  (buf1[1] & 0xFF000000) |   buf1[2];
-              buf1[1] =  (buf1[1] & 0x00FFFFFF) |  (buf1[2] << 24);
-              buf1[2] =  (buf1[2] >> 24);
-              break;
-    case 26:  buf1[2] = ((buf1[2] << 8) & 0x0000FF00) | ((buf1[2] >> 8) & 0x000000FF);
-              break;
-    case 27:  buf1[2] =  (buf1[2] & 0x000000FF) | ((buf1[2] <<  8) & 0x00FF0000) | ((buf1[2] >>  8) & 0x0000FF00);
-              break;
-    case 28:  buf1[2] =  (buf1[2] & 0x0000FFFF) | ((buf1[2] <<  8) & 0xFF000000) | ((buf1[2] >>  8) & 0x00FF0000);
-              break;
-    case 29:  buf1[3] =  (buf1[2] & 0xFF000000) |   buf1[3];
-              buf1[2] =  (buf1[2] & 0x00FFFFFF) |  (buf1[3] << 24);
-              buf1[3] =  (buf1[3] >> 24);
-              break;
-    case 30:  buf1[3] = ((buf1[3] << 8) & 0x0000FF00) | ((buf1[3] >> 8) & 0x000000FF);
-              break;
-    case 31:  buf1[3] =  (buf1[3] & 0x000000FF) |  ((buf1[3] <<  8) & 0x00FF0000) | ((buf1[3] >>  8) & 0x0000FF00);
-              break;
-  }
-  #endif
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_switch_at (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-  if (p1 >= in_len) return (in_len);
-
-  u32 tmp0 = 0;
-  u32 tmp1 = 0;
-
-  #ifdef IS_NV
-  switch (p0)
-  {
-    case  0:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6540);
-              break;
-    case  1:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6541);
-              break;
-    case  2:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6542);
-              break;
-    case  3:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6543);
-              break;
-    case  4:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6540);
-              break;
-    case  5:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6541);
-              break;
-    case  6:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6542);
-              break;
-    case  7:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6543);
-              break;
-    case  8:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6540);
-              break;
-    case  9:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6541);
-              break;
-    case 10:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6542);
-              break;
-    case 11:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6543);
-              break;
-    case 12:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6540);
-              break;
-    case 13:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6541);
-              break;
-    case 14:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6542);
-              break;
-    case 15:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6543);
-              break;
-    case 16:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6540);
-              break;
-    case 17:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6541);
-              break;
-    case 18:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6542);
-              break;
-    case 19:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6543);
-              break;
-    case 20:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6540);
-              break;
-    case 21:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6541);
-              break;
-    case 22:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6542);
-              break;
-    case 23:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6543);
-              break;
-    case 24:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6540);
-              break;
-    case 25:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6541);
-              break;
-    case 26:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6542);
-              break;
-    case 27:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6543);
-              break;
-    case 28:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6540);
-              break;
-    case 29:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6541);
-              break;
-    case 30:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6542);
-              break;
-    case 31:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6543);
-              break;
-  }
-
-  switch (p1)
-  {
-    case  0:  tmp1    = __byte_perm_S (buf0[0],       0, 0x6540);
-              buf0[0] = __byte_perm_S (tmp0,    buf0[0], 0x7650);
-              break;
-    case  1:  tmp1    = __byte_perm_S (buf0[0],       0, 0x6541);
-              buf0[0] = __byte_perm_S (tmp0,    buf0[0], 0x7604);
-              break;
-    case  2:  tmp1    = __byte_perm_S (buf0[0],       0, 0x6542);
-              buf0[0] = __byte_perm_S (tmp0,    buf0[0], 0x7054);
-              break;
-    case  3:  tmp1    = __byte_perm_S (buf0[0],       0, 0x6543);
-              buf0[0] = __byte_perm_S (tmp0,    buf0[0], 0x0654);
-              break;
-    case  4:  tmp1    = __byte_perm_S (buf0[1],       0, 0x6540);
-              buf0[1] = __byte_perm_S (tmp0,    buf0[1], 0x7650);
-              break;
-    case  5:  tmp1    = __byte_perm_S (buf0[1],       0, 0x6541);
-              buf0[1] = __byte_perm_S (tmp0,    buf0[1], 0x7604);
-              break;
-    case  6:  tmp1    = __byte_perm_S (buf0[1],       0, 0x6542);
-              buf0[1] = __byte_perm_S (tmp0,    buf0[1], 0x7054);
-              break;
-    case  7:  tmp1    = __byte_perm_S (buf0[1],       0, 0x6543);
-              buf0[1] = __byte_perm_S (tmp0,    buf0[1], 0x0654);
-              break;
-    case  8:  tmp1    = __byte_perm_S (buf0[2],       0, 0x6540);
-              buf0[2] = __byte_perm_S (tmp0,    buf0[2], 0x7650);
-              break;
-    case  9:  tmp1    = __byte_perm_S (buf0[2],       0, 0x6541);
-              buf0[2] = __byte_perm_S (tmp0,    buf0[2], 0x7604);
-              break;
-    case 10:  tmp1    = __byte_perm_S (buf0[2],       0, 0x6542);
-              buf0[2] = __byte_perm_S (tmp0,    buf0[2], 0x7054);
-              break;
-    case 11:  tmp1    = __byte_perm_S (buf0[2],       0, 0x6543);
-              buf0[2] = __byte_perm_S (tmp0,    buf0[2], 0x0654);
-              break;
-    case 12:  tmp1    = __byte_perm_S (buf0[3],       0, 0x6540);
-              buf0[3] = __byte_perm_S (tmp0,    buf0[3], 0x7650);
-              break;
-    case 13:  tmp1    = __byte_perm_S (buf0[3],       0, 0x6541);
-              buf0[3] = __byte_perm_S (tmp0,    buf0[3], 0x7604);
-              break;
-    case 14:  tmp1    = __byte_perm_S (buf0[3],       0, 0x6542);
-              buf0[3] = __byte_perm_S (tmp0,    buf0[3], 0x7054);
-              break;
-    case 15:  tmp1    = __byte_perm_S (buf0[3],       0, 0x6543);
-              buf0[3] = __byte_perm_S (tmp0,    buf0[3], 0x0654);
-              break;
-    case 16:  tmp1    = __byte_perm_S (buf1[0],       0, 0x6540);
-              buf1[0] = __byte_perm_S (tmp0,    buf1[0], 0x7650);
-              break;
-    case 17:  tmp1    = __byte_perm_S (buf1[0],       0, 0x6541);
-              buf1[0] = __byte_perm_S (tmp0,    buf1[0], 0x7604);
-              break;
-    case 18:  tmp1    = __byte_perm_S (buf1[0],       0, 0x6542);
-              buf1[0] = __byte_perm_S (tmp0,    buf1[0], 0x7054);
-              break;
-    case 19:  tmp1    = __byte_perm_S (buf1[0],       0, 0x6543);
-              buf1[0] = __byte_perm_S (tmp0,    buf1[0], 0x0654);
-              break;
-    case 20:  tmp1    = __byte_perm_S (buf1[1],       0, 0x6540);
-              buf1[1] = __byte_perm_S (tmp0,    buf1[1], 0x7650);
-              break;
-    case 21:  tmp1    = __byte_perm_S (buf1[1],       0, 0x6541);
-              buf1[1] = __byte_perm_S (tmp0,    buf1[1], 0x7604);
-              break;
-    case 22:  tmp1    = __byte_perm_S (buf1[1],       0, 0x6542);
-              buf1[1] = __byte_perm_S (tmp0,    buf1[1], 0x7054);
-              break;
-    case 23:  tmp1    = __byte_perm_S (buf1[1],       0, 0x6543);
-              buf1[1] = __byte_perm_S (tmp0,    buf1[1], 0x0654);
-              break;
-    case 24:  tmp1    = __byte_perm_S (buf1[2],       0, 0x6540);
-              buf1[2] = __byte_perm_S (tmp0,    buf1[2], 0x7650);
-              break;
-    case 25:  tmp1    = __byte_perm_S (buf1[2],       0, 0x6541);
-              buf1[2] = __byte_perm_S (tmp0,    buf1[2], 0x7604);
-              break;
-    case 26:  tmp1    = __byte_perm_S (buf1[2],       0, 0x6542);
-              buf1[2] = __byte_perm_S (tmp0,    buf1[2], 0x7054);
-              break;
-    case 27:  tmp1    = __byte_perm_S (buf1[2],       0, 0x6543);
-              buf1[2] = __byte_perm_S (tmp0,    buf1[2], 0x0654);
-              break;
-    case 28:  tmp1    = __byte_perm_S (buf1[3],       0, 0x6540);
-              buf1[3] = __byte_perm_S (tmp0,    buf1[3], 0x7650);
-              break;
-    case 29:  tmp1    = __byte_perm_S (buf1[3],       0, 0x6541);
-              buf1[3] = __byte_perm_S (tmp0,    buf1[3], 0x7604);
-              break;
-    case 30:  tmp1    = __byte_perm_S (buf1[3],       0, 0x6542);
-              buf1[3] = __byte_perm_S (tmp0,    buf1[3], 0x7054);
-              break;
-    case 31:  tmp1    = __byte_perm_S (buf1[3],       0, 0x6543);
-              buf1[3] = __byte_perm_S (tmp0,    buf1[3], 0x0654);
-              break;
-  }
-
-  switch (p0)
-  {
-    case  0:  buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7650);
-              break;
-    case  1:  buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7604);
-              break;
-    case  2:  buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x7054);
-              break;
-    case  3:  buf0[0] = __byte_perm_S (tmp1, buf0[0], 0x0654);
-              break;
-    case  4:  buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7650);
-              break;
-    case  5:  buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7604);
-              break;
-    case  6:  buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x7054);
-              break;
-    case  7:  buf0[1] = __byte_perm_S (tmp1, buf0[1], 0x0654);
-              break;
-    case  8:  buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7650);
-              break;
-    case  9:  buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7604);
-              break;
-    case 10:  buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x7054);
-              break;
-    case 11:  buf0[2] = __byte_perm_S (tmp1, buf0[2], 0x0654);
-              break;
-    case 12:  buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7650);
-              break;
-    case 13:  buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7604);
-              break;
-    case 14:  buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x7054);
-              break;
-    case 15:  buf0[3] = __byte_perm_S (tmp1, buf0[3], 0x0654);
-              break;
-    case 16:  buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7650);
-              break;
-    case 17:  buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7604);
-              break;
-    case 18:  buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x7054);
-              break;
-    case 19:  buf1[0] = __byte_perm_S (tmp1, buf1[0], 0x0654);
-              break;
-    case 20:  buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7650);
-              break;
-    case 21:  buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7604);
-              break;
-    case 22:  buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x7054);
-              break;
-    case 23:  buf1[1] = __byte_perm_S (tmp1, buf1[1], 0x0654);
-              break;
-    case 24:  buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7650);
-              break;
-    case 25:  buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7604);
-              break;
-    case 26:  buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x7054);
-              break;
-    case 27:  buf1[2] = __byte_perm_S (tmp1, buf1[2], 0x0654);
-              break;
-    case 28:  buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7650);
-              break;
-    case 29:  buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7604);
-              break;
-    case 30:  buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x7054);
-              break;
-    case 31:  buf1[3] = __byte_perm_S (tmp1, buf1[3], 0x0654);
-              break;
-  }
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  switch (p0)
-  {
-    case  0:  tmp0 = (buf0[0] >>  0) & 0xFF;
-              break;
-    case  1:  tmp0 = (buf0[0] >>  8) & 0xFF;
-              break;
-    case  2:  tmp0 = (buf0[0] >> 16) & 0xFF;
-              break;
-    case  3:  tmp0 = (buf0[0] >> 24) & 0xFF;
-              break;
-    case  4:  tmp0 = (buf0[1] >>  0) & 0xFF;
-              break;
-    case  5:  tmp0 = (buf0[1] >>  8) & 0xFF;
-              break;
-    case  6:  tmp0 = (buf0[1] >> 16) & 0xFF;
-              break;
-    case  7:  tmp0 = (buf0[1] >> 24) & 0xFF;
-              break;
-    case  8:  tmp0 = (buf0[2] >>  0) & 0xFF;
-              break;
-    case  9:  tmp0 = (buf0[2] >>  8) & 0xFF;
-              break;
-    case 10:  tmp0 = (buf0[2] >> 16) & 0xFF;
-              break;
-    case 11:  tmp0 = (buf0[2] >> 24) & 0xFF;
-              break;
-    case 12:  tmp0 = (buf0[3] >>  0) & 0xFF;
-              break;
-    case 13:  tmp0 = (buf0[3] >>  8) & 0xFF;
-              break;
-    case 14:  tmp0 = (buf0[3] >> 16) & 0xFF;
-              break;
-    case 15:  tmp0 = (buf0[3] >> 24) & 0xFF;
-              break;
-    case 16:  tmp0 = (buf1[0] >>  0) & 0xFF;
-              break;
-    case 17:  tmp0 = (buf1[0] >>  8) & 0xFF;
-              break;
-    case 18:  tmp0 = (buf1[0] >> 16) & 0xFF;
-              break;
-    case 19:  tmp0 = (buf1[0] >> 24) & 0xFF;
-              break;
-    case 20:  tmp0 = (buf1[1] >>  0) & 0xFF;
-              break;
-    case 21:  tmp0 = (buf1[1] >>  8) & 0xFF;
-              break;
-    case 22:  tmp0 = (buf1[1] >> 16) & 0xFF;
-              break;
-    case 23:  tmp0 = (buf1[1] >> 24) & 0xFF;
-              break;
-    case 24:  tmp0 = (buf1[2] >>  0) & 0xFF;
-              break;
-    case 25:  tmp0 = (buf1[2] >>  8) & 0xFF;
-              break;
-    case 26:  tmp0 = (buf1[2] >> 16) & 0xFF;
-              break;
-    case 27:  tmp0 = (buf1[2] >> 24) & 0xFF;
-              break;
-    case 28:  tmp0 = (buf1[3] >>  0) & 0xFF;
-              break;
-    case 29:  tmp0 = (buf1[3] >>  8) & 0xFF;
-              break;
-    case 30:  tmp0 = (buf1[3] >> 16) & 0xFF;
-              break;
-    case 31:  tmp0 = (buf1[3] >> 24) & 0xFF;
-              break;
-  }
-
-  switch (p1)
-  {
-    case  0:  tmp1 = (buf0[0] >>  0) & 0xff;
-              buf0[0]  = (buf0[0] & 0xffffff00) | tmp0 <<  0;
-              break;
-    case  1:  tmp1 = (buf0[0] >>  8) & 0xff;
-              buf0[0]  = (buf0[0] & 0xffff00ff) | tmp0 <<  8;
-              break;
-    case  2:  tmp1 = (buf0[0] >> 16) & 0xff;
-              buf0[0]  = (buf0[0] & 0xff00ffff) | tmp0 << 16;
-              break;
-    case  3:  tmp1 = (buf0[0] >> 24) & 0xff;
-              buf0[0]  = (buf0[0] & 0x00ffffff) | tmp0 << 24;
-              break;
-    case  4:  tmp1 = (buf0[1] >>  0) & 0xff;
-              buf0[1]  = (buf0[1] & 0xffffff00) | tmp0 <<  0;
-              break;
-    case  5:  tmp1 = (buf0[1] >>  8) & 0xff;
-              buf0[1]  = (buf0[1] & 0xffff00ff) | tmp0 <<  8;
-              break;
-    case  6:  tmp1 = (buf0[1] >> 16) & 0xff;
-              buf0[1]  = (buf0[1] & 0xff00ffff) | tmp0 << 16;
-              break;
-    case  7:  tmp1 = (buf0[1] >> 24) & 0xff;
-              buf0[1]  = (buf0[1] & 0x00ffffff) | tmp0 << 24;
-              break;
-    case  8:  tmp1 = (buf0[2] >>  0) & 0xff;
-              buf0[2]  = (buf0[2] & 0xffffff00) | tmp0 <<  0;
-              break;
-    case  9:  tmp1 = (buf0[2] >>  8) & 0xff;
-              buf0[2]  = (buf0[2] & 0xffff00ff) | tmp0 <<  8;
-              break;
-    case 10:  tmp1 = (buf0[2] >> 16) & 0xff;
-              buf0[2]  = (buf0[2] & 0xff00ffff) | tmp0 << 16;
-              break;
-    case 11:  tmp1 = (buf0[2] >> 24) & 0xff;
-              buf0[2]  = (buf0[2] & 0x00ffffff) | tmp0 << 24;
-              break;
-    case 12:  tmp1 = (buf0[3] >>  0) & 0xff;
-              buf0[3]  = (buf0[3] & 0xffffff00) | tmp0 <<  0;
-              break;
-    case 13:  tmp1 = (buf0[3] >>  8) & 0xff;
-              buf0[3]  = (buf0[3] & 0xffff00ff) | tmp0 <<  8;
-              break;
-    case 14:  tmp1 = (buf0[3] >> 16) & 0xff;
-              buf0[3]  = (buf0[3] & 0xff00ffff) | tmp0 << 16;
-              break;
-    case 15:  tmp1 = (buf0[3] >> 24) & 0xff;
-              buf0[3]  = (buf0[3] & 0x00ffffff) | tmp0 << 24;
-              break;
-    case 16:  tmp1 = (buf1[0] >>  0) & 0xff;
-              buf1[0]  = (buf1[0] & 0xffffff00) | tmp0 <<  0;
-              break;
-    case 17:  tmp1 = (buf1[0] >>  8) & 0xff;
-              buf1[0]  = (buf1[0] & 0xffff00ff) | tmp0 <<  8;
-              break;
-    case 18:  tmp1 = (buf1[0] >> 16) & 0xff;
-              buf1[0]  = (buf1[0] & 0xff00ffff) | tmp0 << 16;
-              break;
-    case 19:  tmp1 = (buf1[0] >> 24) & 0xff;
-              buf1[0]  = (buf1[0] & 0x00ffffff) | tmp0 << 24;
-              break;
-    case 20:  tmp1 = (buf1[1] >>  0) & 0xff;
-              buf1[1]  = (buf1[1] & 0xffffff00) | tmp0 <<  0;
-              break;
-    case 21:  tmp1 = (buf1[1] >>  8) & 0xff;
-              buf1[1]  = (buf1[1] & 0xffff00ff) | tmp0 <<  8;
-              break;
-    case 22:  tmp1 = (buf1[1] >> 16) & 0xff;
-              buf1[1]  = (buf1[1] & 0xff00ffff) | tmp0 << 16;
-              break;
-    case 23:  tmp1 = (buf1[1] >> 24) & 0xff;
-              buf1[1]  = (buf1[1] & 0x00ffffff) | tmp0 << 24;
-              break;
-    case 24:  tmp1 = (buf1[2] >>  0) & 0xff;
-              buf1[2]  = (buf1[2] & 0xffffff00) | tmp0 <<  0;
-              break;
-    case 25:  tmp1 = (buf1[2] >>  8) & 0xff;
-              buf1[2]  = (buf1[2] & 0xffff00ff) | tmp0 <<  8;
-              break;
-    case 26:  tmp1 = (buf1[2] >> 16) & 0xff;
-              buf1[2]  = (buf1[2] & 0xff00ffff) | tmp0 << 16;
-              break;
-    case 27:  tmp1 = (buf1[2] >> 24) & 0xff;
-              buf1[2]  = (buf1[2] & 0x00ffffff) | tmp0 << 24;
-              break;
-    case 28:  tmp1 = (buf1[3] >>  0) & 0xff;
-              buf1[3]  = (buf1[3] & 0xffffff00) | tmp0 <<  0;
-              break;
-    case 29:  tmp1 = (buf1[3] >>  8) & 0xff;
-              buf1[3]  = (buf1[3] & 0xffff00ff) | tmp0 <<  8;
-              break;
-    case 30:  tmp1 = (buf1[3] >> 16) & 0xff;
-              buf1[3]  = (buf1[3] & 0xff00ffff) | tmp0 << 16;
-              break;
-    case 31:  tmp1 = (buf1[3] >> 24) & 0xff;
-              buf1[3]  = (buf1[3] & 0x00ffffff) | tmp0 << 24;
-              break;
-  }
-
-  switch (p0)
-  {
-    case  0:  buf0[0]  = (buf0[0] & 0xffffff00) | tmp1 <<  0;
-              break;
-    case  1:  buf0[0]  = (buf0[0] & 0xffff00ff) | tmp1 <<  8;
-              break;
-    case  2:  buf0[0]  = (buf0[0] & 0xff00ffff) | tmp1 << 16;
-              break;
-    case  3:  buf0[0]  = (buf0[0] & 0x00ffffff) | tmp1 << 24;
-              break;
-    case  4:  buf0[1]  = (buf0[1] & 0xffffff00) | tmp1 <<  0;
-              break;
-    case  5:  buf0[1]  = (buf0[1] & 0xffff00ff) | tmp1 <<  8;
-              break;
-    case  6:  buf0[1]  = (buf0[1] & 0xff00ffff) | tmp1 << 16;
-              break;
-    case  7:  buf0[1]  = (buf0[1] & 0x00ffffff) | tmp1 << 24;
-              break;
-    case  8:  buf0[2]  = (buf0[2] & 0xffffff00) | tmp1 <<  0;
-              break;
-    case  9:  buf0[2]  = (buf0[2] & 0xffff00ff) | tmp1 <<  8;
-              break;
-    case 10:  buf0[2]  = (buf0[2] & 0xff00ffff) | tmp1 << 16;
-              break;
-    case 11:  buf0[2]  = (buf0[2] & 0x00ffffff) | tmp1 << 24;
-              break;
-    case 12:  buf0[3]  = (buf0[3] & 0xffffff00) | tmp1 <<  0;
-              break;
-    case 13:  buf0[3]  = (buf0[3] & 0xffff00ff) | tmp1 <<  8;
-              break;
-    case 14:  buf0[3]  = (buf0[3] & 0xff00ffff) | tmp1 << 16;
-              break;
-    case 15:  buf0[3]  = (buf0[3] & 0x00ffffff) | tmp1 << 24;
-              break;
-    case 16:  buf1[0]  = (buf1[0] & 0xffffff00) | tmp1 <<  0;
-              break;
-    case 17:  buf1[0]  = (buf1[0] & 0xffff00ff) | tmp1 <<  8;
-              break;
-    case 18:  buf1[0]  = (buf1[0] & 0xff00ffff) | tmp1 << 16;
-              break;
-    case 19:  buf1[0]  = (buf1[0] & 0x00ffffff) | tmp1 << 24;
-              break;
-    case 20:  buf1[1]  = (buf1[1] & 0xffffff00) | tmp1 <<  0;
-              break;
-    case 21:  buf1[1]  = (buf1[1] & 0xffff00ff) | tmp1 <<  8;
-              break;
-    case 22:  buf1[1]  = (buf1[1] & 0xff00ffff) | tmp1 << 16;
-              break;
-    case 23:  buf1[1]  = (buf1[1] & 0x00ffffff) | tmp1 << 24;
-              break;
-    case 24:  buf1[2]  = (buf1[2] & 0xffffff00) | tmp1 <<  0;
-              break;
-    case 25:  buf1[2]  = (buf1[2] & 0xffff00ff) | tmp1 <<  8;
-              break;
-    case 26:  buf1[2]  = (buf1[2] & 0xff00ffff) | tmp1 << 16;
-              break;
-    case 27:  buf1[2]  = (buf1[2] & 0x00ffffff) | tmp1 << 24;
-              break;
-    case 28:  buf1[3]  = (buf1[3] & 0xffffff00) | tmp1 <<  0;
-              break;
-    case 29:  buf1[3]  = (buf1[3] & 0xffff00ff) | tmp1 <<  8;
-              break;
-    case 30:  buf1[3]  = (buf1[3] & 0xff00ffff) | tmp1 << 16;
-              break;
-    case 31:  buf1[3]  = (buf1[3] & 0x00ffffff) | tmp1 << 24;
-              break;
-  }
-  #endif
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_chr_shiftl (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  const u32 mr = 0xffu << ((p0 & 3) * 8);
-  const u32 ml = ~mr;
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) << 1) & mr); break;
-    case  1:  buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) << 1) & mr); break;
-    case  2:  buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) << 1) & mr); break;
-    case  3:  buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) << 1) & mr); break;
-    case  4:  buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) << 1) & mr); break;
-    case  5:  buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) << 1) & mr); break;
-    case  6:  buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) << 1) & mr); break;
-    case  7:  buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) << 1) & mr); break;
-  }
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_chr_shiftr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  const u32 mr = 0xffu << ((p0 & 3) * 8);
-  const u32 ml = ~mr;
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) >> 1) & mr); break;
-    case  1:  buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) >> 1) & mr); break;
-    case  2:  buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) >> 1) & mr); break;
-    case  3:  buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) >> 1) & mr); break;
-    case  4:  buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) >> 1) & mr); break;
-    case  5:  buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) >> 1) & mr); break;
-    case  6:  buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) >> 1) & mr); break;
-    case  7:  buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) >> 1) & mr); break;
-  }
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_chr_incr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  const u32 mr = 0xffu << ((p0 & 3) * 8);
-  const u32 ml = ~mr;
-
-  const u32 n = 0x01010101 & mr;
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) + n) & mr); break;
-    case  1:  buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) + n) & mr); break;
-    case  2:  buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) + n) & mr); break;
-    case  3:  buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) + n) & mr); break;
-    case  4:  buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) + n) & mr); break;
-    case  5:  buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) + n) & mr); break;
-    case  6:  buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) + n) & mr); break;
-    case  7:  buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) + n) & mr); break;
-  }
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_chr_decr (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 >= in_len) return (in_len);
-
-  const u32 mr = 0xffu << ((p0 & 3) * 8);
-  const u32 ml = ~mr;
-
-  const u32 n = 0x01010101 & mr;
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] = (buf0[0] & ml) | (((buf0[0] & mr) - n) & mr); break;
-    case  1:  buf0[1] = (buf0[1] & ml) | (((buf0[1] & mr) - n) & mr); break;
-    case  2:  buf0[2] = (buf0[2] & ml) | (((buf0[2] & mr) - n) & mr); break;
-    case  3:  buf0[3] = (buf0[3] & ml) | (((buf0[3] & mr) - n) & mr); break;
-    case  4:  buf1[0] = (buf1[0] & ml) | (((buf1[0] & mr) - n) & mr); break;
-    case  5:  buf1[1] = (buf1[1] & ml) | (((buf1[1] & mr) - n) & mr); break;
-    case  6:  buf1[2] = (buf1[2] & ml) | (((buf1[2] & mr) - n) & mr); break;
-    case  7:  buf1[3] = (buf1[3] & ml) | (((buf1[3] & mr) - n) & mr); break;
-  }
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_replace_np1 (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if ((p0 + 1) >= in_len) return (in_len);
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  lshift_block (buf0, buf1, tib40, tib41);
-
-  const u32 mr = 0xffu << ((p0 & 3) * 8);
-  const u32 ml = ~mr;
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] = (buf0[0] & ml) | (tib40[0] & mr); break;
-    case  1:  buf0[1] = (buf0[1] & ml) | (tib40[1] & mr); break;
-    case  2:  buf0[2] = (buf0[2] & ml) | (tib40[2] & mr); break;
-    case  3:  buf0[3] = (buf0[3] & ml) | (tib40[3] & mr); break;
-    case  4:  buf1[0] = (buf1[0] & ml) | (tib41[0] & mr); break;
-    case  5:  buf1[1] = (buf1[1] & ml) | (tib41[1] & mr); break;
-    case  6:  buf1[2] = (buf1[2] & ml) | (tib41[2] & mr); break;
-    case  7:  buf1[3] = (buf1[3] & ml) | (tib41[3] & mr); break;
-  }
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_replace_nm1 (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 == 0) return (in_len);
-
-  if (p0 >= in_len) return (in_len);
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  rshift_block (buf0, buf1, tib40, tib41);
-
-  const u32 mr = 0xffu << ((p0 & 3) * 8);
-  const u32 ml = ~mr;
-
-  switch (p0 / 4)
-  {
-    case  0:  buf0[0] = (buf0[0] & ml) | (tib40[0] & mr); break;
-    case  1:  buf0[1] = (buf0[1] & ml) | (tib40[1] & mr); break;
-    case  2:  buf0[2] = (buf0[2] & ml) | (tib40[2] & mr); break;
-    case  3:  buf0[3] = (buf0[3] & ml) | (tib40[3] & mr); break;
-    case  4:  buf1[0] = (buf1[0] & ml) | (tib41[0] & mr); break;
-    case  5:  buf1[1] = (buf1[1] & ml) | (tib41[1] & mr); break;
-    case  6:  buf1[2] = (buf1[2] & ml) | (tib41[2] & mr); break;
-    case  7:  buf1[3] = (buf1[3] & ml) | (tib41[3] & mr); break;
-  }
-
-  return in_len;
-}
-
-inline u32 rule_op_mangle_dupeblock_first (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 > in_len) return (in_len);
-
-  if ((in_len + p0) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  tib40[0] = buf0[0];
-  tib40[1] = buf0[1];
-  tib40[2] = buf0[2];
-  tib40[3] = buf0[3];
-  tib41[0] = buf1[0];
-  tib41[1] = buf1[1];
-  tib41[2] = buf1[2];
-  tib41[3] = buf1[3];
-
-  truncate_right (tib40, tib41, p0);
-
-  rshift_block_N (buf0, buf1, buf0, buf1, p0);
-
-  buf0[0] |= tib40[0];
-  buf0[1] |= tib40[1];
-  buf0[2] |= tib40[2];
-  buf0[3] |= tib40[3];
-  buf1[0] |= tib41[0];
-  buf1[1] |= tib41[1];
-  buf1[2] |= tib41[2];
-  buf1[3] |= tib41[3];
-
-  out_len += p0;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_dupeblock_last (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  if (p0 > in_len) return (in_len);
-
-  if ((in_len + p0) >= 32) return (in_len);
-
-  u32 out_len = in_len;
-
-  u32 tib40[4];
-  u32 tib41[4];
-
-  rshift_block_N (buf0, buf1, tib40, tib41, p0);
-
-  truncate_left (tib40, tib41, out_len);
-
-  buf0[0] |= tib40[0];
-  buf0[1] |= tib40[1];
-  buf0[2] |= tib40[2];
-  buf0[3] |= tib40[3];
-  buf1[0] |= tib41[0];
-  buf1[1] |= tib41[1];
-  buf1[2] |= tib41[2];
-  buf1[3] |= tib41[3];
-
-  out_len += p0;
-
-  return out_len;
-}
-
-inline u32 rule_op_mangle_title (const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  buf0[0] |= (generate_cmask (buf0[0]));
-  buf0[1] |= (generate_cmask (buf0[1]));
-  buf0[2] |= (generate_cmask (buf0[2]));
-  buf0[3] |= (generate_cmask (buf0[3]));
-  buf1[0] |= (generate_cmask (buf1[0]));
-  buf1[1] |= (generate_cmask (buf1[1]));
-  buf1[2] |= (generate_cmask (buf1[2]));
-  buf1[3] |= (generate_cmask (buf1[3]));
-
-  #ifdef IS_NV
-  buf0[0] &= ~(0x00000020 & generate_cmask (buf0[0]));
-
-  for (u32 i = 0; i < in_len; i++)
-  {
-    u32 tmp0;
-    u32 tmp1;
-
-    switch (i)
-    {
-      case  0:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6540);
-                tmp1 = ~(0x00002000 & generate_cmask (buf0[0])); break;
-      case  1:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6541);
-                tmp1 = ~(0x00200000 & generate_cmask (buf0[0])); break;
-      case  2:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6542);
-                tmp1 = ~(0x20000000 & generate_cmask (buf0[0])); break;
-      case  3:  tmp0 = __byte_perm_S (buf0[0], 0, 0x6543);
-                tmp1 = ~(0x00000020 & generate_cmask (buf0[1])); break;
-      case  4:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6540);
-                tmp1 = ~(0x00002000 & generate_cmask (buf0[1])); break;
-      case  5:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6541);
-                tmp1 = ~(0x00200000 & generate_cmask (buf0[1])); break;
-      case  6:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6542);
-                tmp1 = ~(0x20000000 & generate_cmask (buf0[1])); break;
-      case  7:  tmp0 = __byte_perm_S (buf0[1], 0, 0x6543);
-                tmp1 = ~(0x00000020 & generate_cmask (buf0[2])); break;
-      case  8:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6540);
-                tmp1 = ~(0x00002000 & generate_cmask (buf0[2])); break;
-      case  9:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6541);
-                tmp1 = ~(0x00200000 & generate_cmask (buf0[2])); break;
-      case 10:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6542);
-                tmp1 = ~(0x20000000 & generate_cmask (buf0[2])); break;
-      case 11:  tmp0 = __byte_perm_S (buf0[2], 0, 0x6543);
-                tmp1 = ~(0x00000020 & generate_cmask (buf0[3])); break;
-      case 12:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6540);
-                tmp1 = ~(0x00002000 & generate_cmask (buf0[3])); break;
-      case 13:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6541);
-                tmp1 = ~(0x00200000 & generate_cmask (buf0[3])); break;
-      case 14:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6542);
-                tmp1 = ~(0x20000000 & generate_cmask (buf0[3])); break;
-      case 15:  tmp0 = __byte_perm_S (buf0[3], 0, 0x6543);
-                tmp1 = ~(0x00000020 & generate_cmask (buf1[0])); break;
-      case 16:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6540);
-                tmp1 = ~(0x00002000 & generate_cmask (buf1[0])); break;
-      case 17:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6541);
-                tmp1 = ~(0x00200000 & generate_cmask (buf1[0])); break;
-      case 18:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6542);
-                tmp1 = ~(0x20000000 & generate_cmask (buf1[0])); break;
-      case 19:  tmp0 = __byte_perm_S (buf1[0], 0, 0x6543);
-                tmp1 = ~(0x00000020 & generate_cmask (buf1[1])); break;
-      case 20:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6540);
-                tmp1 = ~(0x00002000 & generate_cmask (buf1[1])); break;
-      case 21:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6541);
-                tmp1 = ~(0x00200000 & generate_cmask (buf1[1])); break;
-      case 22:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6542);
-                tmp1 = ~(0x20000000 & generate_cmask (buf1[1])); break;
-      case 23:  tmp0 = __byte_perm_S (buf1[1], 0, 0x6543);
-                tmp1 = ~(0x00000020 & generate_cmask (buf1[2])); break;
-      case 24:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6540);
-                tmp1 = ~(0x00002000 & generate_cmask (buf1[2])); break;
-      case 25:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6541);
-                tmp1 = ~(0x00200000 & generate_cmask (buf1[2])); break;
-      case 26:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6542);
-                tmp1 = ~(0x20000000 & generate_cmask (buf1[2])); break;
-      case 27:  tmp0 = __byte_perm_S (buf1[2], 0, 0x6543);
-                tmp1 = ~(0x00000020 & generate_cmask (buf1[3])); break;
-      case 28:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6540);
-                tmp1 = ~(0x00002000 & generate_cmask (buf1[3])); break;
-      case 29:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6541);
-                tmp1 = ~(0x00200000 & generate_cmask (buf1[3])); break;
-      case 30:  tmp0 = __byte_perm_S (buf1[3], 0, 0x6542);
-                tmp1 = ~(0x20000000 & generate_cmask (buf1[3])); break;
-    }
-
-    if (i < 3)
-    {
-      if (tmp0   == ' ') buf0[0]   &= tmp1  ;
-    }
-    else if (i < 7)
-    {
-      if (tmp0   == ' ') buf0[1]   &= tmp1  ;
-    }
-    else if (i < 11)
-    {
-      if (tmp0   == ' ') buf0[2]   &= tmp1  ;
-    }
-    else if (i < 15)
-    {
-      if (tmp0   == ' ') buf0[3]   &= tmp1  ;
-    }
-    else if (i < 19)
-    {
-      if (tmp0   == ' ') buf1[0]   &= tmp1  ;
-    }
-    else if (i < 23)
-    {
-      if (tmp0   == ' ') buf1[1]   &= tmp1  ;
-    }
-    else if (i < 27)
-    {
-      if (tmp0   == ' ') buf1[2]   &= tmp1  ;
-    }
-    else if (i < 31)
-    {
-      if (tmp0   == ' ') buf1[3]   &= tmp1  ;
-    }
-  }
-  #endif
-
-  #if defined IS_AMD || defined IS_GENERIC
-  u32 tib40[4];
-  u32 tib41[4];
-
-  const uchar4 tmp0 = (uchar4) (' ');
-  const uchar4 tmp1 = (uchar4) (0x00);
-  const uchar4 tmp2 = (uchar4) (0xff);
-
-  uchar4 tmp;
-
-  tmp = as_uchar4 (buf0[0]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[0] = as_uint (tmp);
-  tmp = as_uchar4 (buf0[1]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[1] = as_uint (tmp);
-  tmp = as_uchar4 (buf0[2]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[2] = as_uint (tmp);
-  tmp = as_uchar4 (buf0[3]); tmp = select (tmp1, tmp2, tmp == tmp0); tib40[3] = as_uint (tmp);
-  tmp = as_uchar4 (buf1[0]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[0] = as_uint (tmp);
-  tmp = as_uchar4 (buf1[1]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[1] = as_uint (tmp);
-  tmp = as_uchar4 (buf1[2]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[2] = as_uint (tmp);
-  tmp = as_uchar4 (buf1[3]); tmp = select (tmp1, tmp2, tmp == tmp0); tib41[3] = as_uint (tmp);
-
-  rshift_block (tib40, tib41, tib40, tib41); tib40[0] |= 0xff;
-
-  buf0[0] &= ~(generate_cmask (buf0[0]) & tib40[0]);
-  buf0[1] &= ~(generate_cmask (buf0[1]) & tib40[1]);
-  buf0[2] &= ~(generate_cmask (buf0[2]) & tib40[2]);
-  buf0[3] &= ~(generate_cmask (buf0[3]) & tib40[3]);
-  buf1[0] &= ~(generate_cmask (buf1[0]) & tib41[0]);
-  buf1[1] &= ~(generate_cmask (buf1[1]) & tib41[1]);
-  buf1[2] &= ~(generate_cmask (buf1[2]) & tib41[2]);
-  buf1[3] &= ~(generate_cmask (buf1[3]) & tib41[3]);
-  #endif
-
-  return in_len;
-}
-
-inline u32 apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len)
-{
-  u32 out_len = in_len;
-
-  switch (name)
-  {
-    case RULE_OP_MANGLE_LREST:            out_len = rule_op_mangle_lrest            (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_UREST:            out_len = rule_op_mangle_urest            (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_LREST_UFIRST:     out_len = rule_op_mangle_lrest_ufirst     (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_UREST_LFIRST:     out_len = rule_op_mangle_urest_lfirst     (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_TREST:            out_len = rule_op_mangle_trest            (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_TOGGLE_AT:        out_len = rule_op_mangle_toggle_at        (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_REVERSE:          out_len = rule_op_mangle_reverse          (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DUPEWORD:         out_len = rule_op_mangle_dupeword         (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DUPEWORD_TIMES:   out_len = rule_op_mangle_dupeword_times   (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_REFLECT:          out_len = rule_op_mangle_reflect          (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_APPEND:           out_len = rule_op_mangle_append           (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_PREPEND:          out_len = rule_op_mangle_prepend          (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_ROTATE_LEFT:      out_len = rule_op_mangle_rotate_left      (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_ROTATE_RIGHT:     out_len = rule_op_mangle_rotate_right     (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DELETE_FIRST:     out_len = rule_op_mangle_delete_first     (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DELETE_LAST:      out_len = rule_op_mangle_delete_last      (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DELETE_AT:        out_len = rule_op_mangle_delete_at        (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_EXTRACT:          out_len = rule_op_mangle_extract          (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_OMIT:             out_len = rule_op_mangle_omit             (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_INSERT:           out_len = rule_op_mangle_insert           (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_OVERSTRIKE:       out_len = rule_op_mangle_overstrike       (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_TRUNCATE_AT:      out_len = rule_op_mangle_truncate_at      (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_REPLACE:          out_len = rule_op_mangle_replace          (p0, p1, buf0, buf1, out_len); break;
-    //case RULE_OP_MANGLE_PURGECHAR:        out_len = rule_op_mangle_purgechar        (p0, p1, buf0, buf1, out_len); break;
-    //case RULE_OP_MANGLE_TOGGLECASE_REC:   out_len = rule_op_mangle_togglecase_rec   (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DUPECHAR_FIRST:   out_len = rule_op_mangle_dupechar_first   (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DUPECHAR_LAST:    out_len = rule_op_mangle_dupechar_last    (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DUPECHAR_ALL:     out_len = rule_op_mangle_dupechar_all     (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_SWITCH_FIRST:     out_len = rule_op_mangle_switch_first     (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_SWITCH_LAST:      out_len = rule_op_mangle_switch_last      (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_SWITCH_AT:        out_len = rule_op_mangle_switch_at        (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_CHR_SHIFTL:       out_len = rule_op_mangle_chr_shiftl       (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_CHR_SHIFTR:       out_len = rule_op_mangle_chr_shiftr       (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_CHR_INCR:         out_len = rule_op_mangle_chr_incr         (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_CHR_DECR:         out_len = rule_op_mangle_chr_decr         (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_REPLACE_NP1:      out_len = rule_op_mangle_replace_np1      (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_REPLACE_NM1:      out_len = rule_op_mangle_replace_nm1      (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DUPEBLOCK_FIRST:  out_len = rule_op_mangle_dupeblock_first  (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_DUPEBLOCK_LAST:   out_len = rule_op_mangle_dupeblock_last   (p0, p1, buf0, buf1, out_len); break;
-    case RULE_OP_MANGLE_TITLE:            out_len = rule_op_mangle_title            (p0, p1, buf0, buf1, out_len); break;
-  }
-
-  return out_len;
-}
-
-inline u32 apply_rules (const __global u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len)
-{
-  u32 out_len = len;
-
-  for (u32 i = 0; cmds[i] != 0; i++)
-  {
-    const u32 cmd = cmds[i];
-
-    const u32 name = (cmd >>  0) & 0xff;
-    const u32 p0   = (cmd >>  8) & 0xff;
-    const u32 p1   = (cmd >> 16) & 0xff;
-
-    out_len = apply_rule (name, p0, p1, buf0, buf1, out_len);
-  }
-
-  return out_len;
-}
-
-inline u32x apply_rules_vect (const u32 pw_buf0[4], const u32 pw_buf1[4], const u32 pw_len, const __global kernel_rule_t *rules_buf, const u32 il_pos, u32x w0[4], u32x w1[4])
-{
-  #if VECT_SIZE == 1
-
-  w0[0] = pw_buf0[0];
-  w0[1] = pw_buf0[1];
-  w0[2] = pw_buf0[2];
-  w0[3] = pw_buf0[3];
-  w1[0] = pw_buf1[0];
-  w1[1] = pw_buf1[1];
-  w1[2] = pw_buf1[2];
-  w1[3] = pw_buf1[3];
-
-  return apply_rules (rules_buf[il_pos].cmds, w0, w1, pw_len);
-
-  #else
-
-  u32x out_len = 0;
-
-  #ifdef _unroll
-  #pragma unroll
-  #endif
-  for (int i = 0; i < VECT_SIZE; i++)
-  {
-    u32 tmp0[4];
-    u32 tmp1[4];
-
-    tmp0[0] = pw_buf0[0];
-    tmp0[1] = pw_buf0[1];
-    tmp0[2] = pw_buf0[2];
-    tmp0[3] = pw_buf0[3];
-    tmp1[0] = pw_buf1[0];
-    tmp1[1] = pw_buf1[1];
-    tmp1[2] = pw_buf1[2];
-    tmp1[3] = pw_buf1[3];
-
-    const u32 tmp_len = apply_rules (rules_buf[il_pos + i].cmds, tmp0, tmp1, pw_len);
-
-    switch (i)
-    {
-      #if VECT_SIZE >= 2
-      case 0:
-        w0[0].s0 = tmp0[0];
-        w0[1].s0 = tmp0[1];
-        w0[2].s0 = tmp0[2];
-        w0[3].s0 = tmp0[3];
-        w1[0].s0 = tmp1[0];
-        w1[1].s0 = tmp1[1];
-        w1[2].s0 = tmp1[2];
-        w1[3].s0 = tmp1[3];
-        out_len.s0 = tmp_len;
-        break;
-
-      case 1:
-        w0[0].s1 = tmp0[0];
-        w0[1].s1 = tmp0[1];
-        w0[2].s1 = tmp0[2];
-        w0[3].s1 = tmp0[3];
-        w1[0].s1 = tmp1[0];
-        w1[1].s1 = tmp1[1];
-        w1[2].s1 = tmp1[2];
-        w1[3].s1 = tmp1[3];
-        out_len.s1 = tmp_len;
-        break;
-      #endif
-
-      #if VECT_SIZE >= 4
-      case 2:
-        w0[0].s2 = tmp0[0];
-        w0[1].s2 = tmp0[1];
-        w0[2].s2 = tmp0[2];
-        w0[3].s2 = tmp0[3];
-        w1[0].s2 = tmp1[0];
-        w1[1].s2 = tmp1[1];
-        w1[2].s2 = tmp1[2];
-        w1[3].s2 = tmp1[3];
-        out_len.s2 = tmp_len;
-        break;
-
-      case 3:
-        w0[0].s3 = tmp0[0];
-        w0[1].s3 = tmp0[1];
-        w0[2].s3 = tmp0[2];
-        w0[3].s3 = tmp0[3];
-        w1[0].s3 = tmp1[0];
-        w1[1].s3 = tmp1[1];
-        w1[2].s3 = tmp1[2];
-        w1[3].s3 = tmp1[3];
-        out_len.s3 = tmp_len;
-        break;
-      #endif
-
-      #if VECT_SIZE >= 8
-      case 4:
-        w0[0].s4 = tmp0[0];
-        w0[1].s4 = tmp0[1];
-        w0[2].s4 = tmp0[2];
-        w0[3].s4 = tmp0[3];
-        w1[0].s4 = tmp1[0];
-        w1[1].s4 = tmp1[1];
-        w1[2].s4 = tmp1[2];
-        w1[3].s4 = tmp1[3];
-        out_len.s4 = tmp_len;
-        break;
-
-      case 5:
-        w0[0].s5 = tmp0[0];
-        w0[1].s5 = tmp0[1];
-        w0[2].s5 = tmp0[2];
-        w0[3].s5 = tmp0[3];
-        w1[0].s5 = tmp1[0];
-        w1[1].s5 = tmp1[1];
-        w1[2].s5 = tmp1[2];
-        w1[3].s5 = tmp1[3];
-        out_len.s5 = tmp_len;
-        break;
-
-      case 6:
-        w0[0].s6 = tmp0[0];
-        w0[1].s6 = tmp0[1];
-        w0[2].s6 = tmp0[2];
-        w0[3].s6 = tmp0[3];
-        w1[0].s6 = tmp1[0];
-        w1[1].s6 = tmp1[1];
-        w1[2].s6 = tmp1[2];
-        w1[3].s6 = tmp1[3];
-        out_len.s6 = tmp_len;
-        break;
-
-      case 7:
-        w0[0].s7 = tmp0[0];
-        w0[1].s7 = tmp0[1];
-        w0[2].s7 = tmp0[2];
-        w0[3].s7 = tmp0[3];
-        w1[0].s7 = tmp1[0];
-        w1[1].s7 = tmp1[1];
-        w1[2].s7 = tmp1[2];
-        w1[3].s7 = tmp1[3];
-        out_len.s7 = tmp_len;
-        break;
-      #endif
-
-      #if VECT_SIZE >= 16
-      case 8:
-        w0[0].s8 = tmp0[0];
-        w0[1].s8 = tmp0[1];
-        w0[2].s8 = tmp0[2];
-        w0[3].s8 = tmp0[3];
-        w1[0].s8 = tmp1[0];
-        w1[1].s8 = tmp1[1];
-        w1[2].s8 = tmp1[2];
-        w1[3].s8 = tmp1[3];
-        out_len.s8 = tmp_len;
-        break;
-
-      case 9:
-        w0[0].s9 = tmp0[0];
-        w0[1].s9 = tmp0[1];
-        w0[2].s9 = tmp0[2];
-        w0[3].s9 = tmp0[3];
-        w1[0].s9 = tmp1[0];
-        w1[1].s9 = tmp1[1];
-        w1[2].s9 = tmp1[2];
-        w1[3].s9 = tmp1[3];
-        out_len.s9 = tmp_len;
-        break;
-
-      case 10:
-        w0[0].sa = tmp0[0];
-        w0[1].sa = tmp0[1];
-        w0[2].sa = tmp0[2];
-        w0[3].sa = tmp0[3];
-        w1[0].sa = tmp1[0];
-        w1[1].sa = tmp1[1];
-        w1[2].sa = tmp1[2];
-        w1[3].sa = tmp1[3];
-        out_len.sa = tmp_len;
-        break;
-
-      case 11:
-        w0[0].sb = tmp0[0];
-        w0[1].sb = tmp0[1];
-        w0[2].sb = tmp0[2];
-        w0[3].sb = tmp0[3];
-        w1[0].sb = tmp1[0];
-        w1[1].sb = tmp1[1];
-        w1[2].sb = tmp1[2];
-        w1[3].sb = tmp1[3];
-        out_len.sb = tmp_len;
-        break;
-
-      case 12:
-        w0[0].sc = tmp0[0];
-        w0[1].sc = tmp0[1];
-        w0[2].sc = tmp0[2];
-        w0[3].sc = tmp0[3];
-        w1[0].sc = tmp1[0];
-        w1[1].sc = tmp1[1];
-        w1[2].sc = tmp1[2];
-        w1[3].sc = tmp1[3];
-        out_len.sc = tmp_len;
-        break;
-
-      case 13:
-        w0[0].sd = tmp0[0];
-        w0[1].sd = tmp0[1];
-        w0[2].sd = tmp0[2];
-        w0[3].sd = tmp0[3];
-        w1[0].sd = tmp1[0];
-        w1[1].sd = tmp1[1];
-        w1[2].sd = tmp1[2];
-        w1[3].sd = tmp1[3];
-        out_len.sd = tmp_len;
-        break;
-
-      case 14:
-        w0[0].se = tmp0[0];
-        w0[1].se = tmp0[1];
-        w0[2].se = tmp0[2];
-        w0[3].se = tmp0[3];
-        w1[0].se = tmp1[0];
-        w1[1].se = tmp1[1];
-        w1[2].se = tmp1[2];
-        w1[3].se = tmp1[3];
-        out_len.se = tmp_len;
-        break;
-
-      case 15:
-        w0[0].sf = tmp0[0];
-        w0[1].sf = tmp0[1];
-        w0[2].sf = tmp0[2];
-        w0[3].sf = tmp0[3];
-        w1[0].sf = tmp1[0];
-        w1[1].sf = tmp1[1];
-        w1[2].sf = tmp1[2];
-        w1[3].sf = tmp1[3];
-        out_len.sf = tmp_len;
-        break;
-      #endif
-    }
-  }
-
-  return out_len;
-
-  #endif
-}

diff --git a/OpenCL/simd.c b/OpenCL/simd.c
deleted file mode 100644 (file)
index 882e320..0000000
--- a/OpenCL/simd.c
+++ /dev/null
@@ -1,1164 +0,0 @@
-/**
- * Authors.....: Jens Steube <jens.steube@gmail.com>
- *               magnum <john.magnum@hushmail.com>
- *
- * License.....: MIT
- */
-
-// vliw1
-
-#if VECT_SIZE == 1
-
-#define MATCHES_ONE_VV(a,b) ((a) == (b))
-#define MATCHES_ONE_VS(a,b) ((a) == (b))
-
-#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  if (((h0) == search[0]) && ((h1) == search[1]) && ((h2) == search[2]) && ((h3) == search[3]))             \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)                                                    \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos);                       \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  const u32 digest_tp0[4] = { h0, h1, h2, h3 };                                                             \
-                                                                                                            \
-  if (check (digest_tp0,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (atomic_inc (&hashes_shown[final_hash_pos]) == 0)                                                  \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos);            \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#endif
-
-// vliw2
-
-#define vector_accessible(p,c,e) (((p) + (e)) < (c))
-
-#if VECT_SIZE == 2
-
-#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1))
-#define MATCHES_ONE_VS(a,b) (((a).s0 == (b)   ) || ((a).s1 == (b)   ))
-
-#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 };                                                 \
-  const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 };                                                 \
-                                                                                                            \
-  if (check (digest_tp0,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp1,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#endif
-
-// vliw4
-
-#if VECT_SIZE == 4
-
-#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3))
-#define MATCHES_ONE_VS(a,b) (((a).s0 == (b)   ) || ((a).s1 == (b)   ) || ((a).s2 == (b)   ) || ((a).s3 == (b)   ))
-
-#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 };                                                 \
-  const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 };                                                 \
-  const u32 digest_tp2[4] = { h0.s2, h1.s2, h2.s2, h3.s2 };                                                 \
-  const u32 digest_tp3[4] = { h0.s3, h1.s3, h2.s3, h3.s3 };                                                 \
-                                                                                                            \
-  if (check (digest_tp0,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp1,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp2,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp2, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp3,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp3, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#endif
-
-// vliw8
-
-#if VECT_SIZE == 8
-
-#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3) || ((a).s4 == (b).s4) || ((a).s5 == (b).s5) || ((a).s6 == (b).s6) || ((a).s7 == (b).s7))
-#define MATCHES_ONE_VS(a,b) (((a).s0 == (b)   ) || ((a).s1 == (b)   ) || ((a).s2 == (b)   ) || ((a).s3 == (b)   ) || ((a).s4 == (b)   ) || ((a).s5 == (b)   ) || ((a).s6 == (b)   ) || ((a).s7 == (b)   ))
-
-#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-  if (((h0).s4 == search[0]) && ((h1).s4 == search[1]) && ((h2).s4 == search[2]) && ((h3).s4 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 4);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s5 == search[0]) && ((h1).s5 == search[1]) && ((h2).s5 == search[2]) && ((h3).s5 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 5);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s6 == search[0]) && ((h1).s6 == search[1]) && ((h2).s6 == search[2]) && ((h3).s6 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 6);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s7 == search[0]) && ((h1).s7 == search[1]) && ((h2).s7 == search[2]) && ((h3).s7 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 7);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  const u32 digest_tp0[4] = { h0.s0, h1.s0, h2.s0, h3.s0 };                                                 \
-  const u32 digest_tp1[4] = { h0.s1, h1.s1, h2.s1, h3.s1 };                                                 \
-  const u32 digest_tp2[4] = { h0.s2, h1.s2, h2.s2, h3.s2 };                                                 \
-  const u32 digest_tp3[4] = { h0.s3, h1.s3, h2.s3, h3.s3 };                                                 \
-  const u32 digest_tp4[4] = { h0.s4, h1.s4, h2.s4, h3.s4 };                                                 \
-  const u32 digest_tp5[4] = { h0.s5, h1.s5, h2.s5, h3.s5 };                                                 \
-  const u32 digest_tp6[4] = { h0.s6, h1.s6, h2.s6, h3.s6 };                                                 \
-  const u32 digest_tp7[4] = { h0.s7, h1.s7, h2.s7, h3.s7 };                                                 \
-                                                                                                            \
-  if (check (digest_tp0,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp0, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp1,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp1, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp2,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp2, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp3,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp3, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-  if (check (digest_tp4,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp4, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 4);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp5,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp5, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 5);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp6,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp6, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 6);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp7,                                                                                    \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp7, digests_cnt, &digests_buf[digests_offset]);                     \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 7);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#endif
-
-// vliw16
-
-#if VECT_SIZE == 16
-
-#define MATCHES_ONE_VV(a,b) (((a).s0 == (b).s0) || ((a).s1 == (b).s1) || ((a).s2 == (b).s2) || ((a).s3 == (b).s3) || ((a).s4 == (b).s4) || ((a).s5 == (b).s5) || ((a).s6 == (b).s6) || ((a).s7 == (b).s7) || ((a).s8 == (b).s8) || ((a).s9 == (b).s9) || ((a).sa == (b).sa) || ((a).sb == (b).sb) || ((a).sc == (b).sc) || ((a).sd == (b).sd) || ((a).se == (b).se) || ((a).sf == (b).sf))
-#define MATCHES_ONE_VS(a,b) (((a).s0 == (b)   ) || ((a).s1 == (b)   ) || ((a).s2 == (b)   ) || ((a).s3 == (b)   ) || ((a).s4 == (b)   ) || ((a).s5 == (b)   ) || ((a).s6 == (b)   ) || ((a).s7 == (b)   ) || ((a).s8 == (b)   ) || ((a).s9 == (b)   ) || ((a).sa == (b)   ) || ((a).sb == (b)   ) || ((a).sc == (b)   ) || ((a).sd == (b)   ) || ((a).se == (b)   ) || ((a).sf == (b)   ))
-
-#define COMPARE_S_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  if (((h0).s0 == search[0]) && ((h1).s0 == search[1]) && ((h2).s0 == search[2]) && ((h3).s0 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 0);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s1 == search[0]) && ((h1).s1 == search[1]) && ((h2).s1 == search[2]) && ((h3).s1 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 1);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s2 == search[0]) && ((h1).s2 == search[1]) && ((h2).s2 == search[2]) && ((h3).s2 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 2);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s3 == search[0]) && ((h1).s3 == search[1]) && ((h2).s3 == search[2]) && ((h3).s3 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 3);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-  if (((h0).s4 == search[0]) && ((h1).s4 == search[1]) && ((h2).s4 == search[2]) && ((h3).s4 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 4);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s5 == search[0]) && ((h1).s5 == search[1]) && ((h2).s5 == search[2]) && ((h3).s5 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 5);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s6 == search[0]) && ((h1).s6 == search[1]) && ((h2).s6 == search[2]) && ((h3).s6 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 6);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s7 == search[0]) && ((h1).s7 == search[1]) && ((h2).s7 == search[2]) && ((h3).s7 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 7);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s8 == search[0]) && ((h1).s8 == search[1]) && ((h2).s8 == search[2]) && ((h3).s8 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 8) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 8);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).s9 == search[0]) && ((h1).s9 == search[1]) && ((h2).s9 == search[2]) && ((h3).s9 == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 9) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))         \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 9);                   \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).sa == search[0]) && ((h1).sa == search[1]) && ((h2).sa == search[2]) && ((h3).sa == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 10) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 10);                  \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).sb == search[0]) && ((h1).sb == search[1]) && ((h2).sb == search[2]) && ((h3).sb == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 11) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 11);                  \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).sc == search[0]) && ((h1).sc == search[1]) && ((h2).sc == search[2]) && ((h3).sc == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 12) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 12);                  \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).sd == search[0]) && ((h1).sd == search[1]) && ((h2).sd == search[2]) && ((h3).sd == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 13) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 13);                  \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).se == search[0]) && ((h1).se == search[1]) && ((h2).se == search[2]) && ((h3).se == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 14) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 14);                  \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (((h0).sf == search[0]) && ((h1).sf == search[1]) && ((h2).sf == search[2]) && ((h3).sf == search[3])) \
-  {                                                                                                         \
-    const u32 final_hash_pos = digests_offset + 0;                                                          \
-                                                                                                            \
-    if (vector_accessible (il_pos, il_cnt, 15) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))        \
-    {                                                                                                       \
-      mark_hash (plains_buf, d_return_buf, salt_pos, 0, final_hash_pos, gid, il_pos + 15);                  \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#define COMPARE_M_SIMD(h0,h1,h2,h3)                                                                         \
-{                                                                                                           \
-  const u32 digest_tp00[4] = { h0.s0, h1.s0, h2.s0, h3.s0 };                                                \
-  const u32 digest_tp01[4] = { h0.s1, h1.s1, h2.s1, h3.s1 };                                                \
-  const u32 digest_tp02[4] = { h0.s2, h1.s2, h2.s2, h3.s2 };                                                \
-  const u32 digest_tp03[4] = { h0.s3, h1.s3, h2.s3, h3.s3 };                                                \
-  const u32 digest_tp04[4] = { h0.s4, h1.s4, h2.s4, h3.s4 };                                                \
-  const u32 digest_tp05[4] = { h0.s5, h1.s5, h2.s5, h3.s5 };                                                \
-  const u32 digest_tp06[4] = { h0.s6, h1.s6, h2.s6, h3.s6 };                                                \
-  const u32 digest_tp07[4] = { h0.s7, h1.s7, h2.s7, h3.s7 };                                                \
-  const u32 digest_tp08[4] = { h0.s8, h1.s8, h2.s8, h3.s8 };                                                \
-  const u32 digest_tp09[4] = { h0.s9, h1.s9, h2.s9, h3.s9 };                                                \
-  const u32 digest_tp10[4] = { h0.sa, h1.sa, h2.sa, h3.sa };                                                \
-  const u32 digest_tp11[4] = { h0.sb, h1.sb, h2.sb, h3.sb };                                                \
-  const u32 digest_tp12[4] = { h0.sc, h1.sc, h2.sc, h3.sc };                                                \
-  const u32 digest_tp13[4] = { h0.sd, h1.sd, h2.sd, h3.sd };                                                \
-  const u32 digest_tp14[4] = { h0.se, h1.se, h2.se, h3.se };                                                \
-  const u32 digest_tp15[4] = { h0.sf, h1.sf, h2.sf, h3.sf };                                                \
-                                                                                                            \
-  if (check (digest_tp00,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp00, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 0) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 0);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp01,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp01, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 1) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 1);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp02,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp02, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 2) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 2);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp03,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp03, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 3) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 3);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp04,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp04, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 4) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 4);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp05,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp05, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 5) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 5);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp06,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp06, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 6) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 6);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp07,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp07, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 7) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 7);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp08,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp08, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 8) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 8);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp09,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp09, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 9) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))       \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 9);        \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp10,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp10, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 10) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 10);       \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp11,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp11, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 11) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 11);       \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp12,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp12, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 12) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 12);       \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp13,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp13, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 13) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 13);       \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp14,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp14, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 14) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 14);       \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-                                                                                                            \
-  if (check (digest_tp15,                                                                                   \
-             bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d,                        \
-             bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d,                        \
-             bitmap_mask,                                                                                   \
-             bitmap_shift1,                                                                                 \
-             bitmap_shift2))                                                                                \
-  {                                                                                                         \
-    int digest_pos = find_hash (digest_tp15, digests_cnt, &digests_buf[digests_offset]);                    \
-                                                                                                            \
-    if (digest_pos != -1)                                                                                   \
-    {                                                                                                       \
-      const u32 final_hash_pos = digests_offset + digest_pos;                                               \
-                                                                                                            \
-      if (vector_accessible (il_pos, il_cnt, 15) && (atomic_inc (&hashes_shown[final_hash_pos]) == 0))      \
-      {                                                                                                     \
-        mark_hash (plains_buf, d_return_buf, salt_pos, digest_pos, final_hash_pos, gid, il_pos + 15);       \
-      }                                                                                                     \
-    }                                                                                                       \
-  }                                                                                                         \
-}
-
-#endif
-
-#define MATCHES_NONE_VV(a,b) !(MATCHES_ONE_VV ((a), (b)))
-#define MATCHES_NONE_VS(a,b) !(MATCHES_ONE_VS ((a), (b)))
-
-// attack-mode 0
-
-inline u32x ix_create_bft (__global bf_t *bfs_buf, const u32 il_pos)
-{
-  #if   VECT_SIZE == 1
-  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i);
-  #elif VECT_SIZE == 2
-  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i);
-  #elif VECT_SIZE == 4
-  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i);
-  #elif VECT_SIZE == 8
-  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i, bfs_buf[il_pos + 4].i, bfs_buf[il_pos + 5].i, bfs_buf[il_pos + 6].i, bfs_buf[il_pos + 7].i);
-  #elif VECT_SIZE == 16
-  const u32x ix = (u32x) (bfs_buf[il_pos + 0].i, bfs_buf[il_pos + 1].i, bfs_buf[il_pos + 2].i, bfs_buf[il_pos + 3].i, bfs_buf[il_pos + 4].i, bfs_buf[il_pos + 5].i, bfs_buf[il_pos + 6].i, bfs_buf[il_pos + 7].i, bfs_buf[il_pos + 8].i, bfs_buf[il_pos + 9].i, bfs_buf[il_pos + 10].i, bfs_buf[il_pos + 11].i, bfs_buf[il_pos + 12].i, bfs_buf[il_pos + 13].i, bfs_buf[il_pos + 14].i, bfs_buf[il_pos + 15].i);
-  #endif
-
-  return ix;
-}
-
-// attack-mode 1
-
-inline u32x pwlenx_create_combt (__global comb_t *combs_buf, const u32 il_pos)
-{
-  #if   VECT_SIZE == 1
-  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len);
-  #elif VECT_SIZE == 2
-  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len);
-  #elif VECT_SIZE == 4
-  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len);
-  #elif VECT_SIZE == 8
-  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len, combs_buf[il_pos + 4].pw_len, combs_buf[il_pos + 5].pw_len, combs_buf[il_pos + 6].pw_len, combs_buf[il_pos + 7].pw_len);
-  #elif VECT_SIZE == 16
-  const u32x pw_lenx = (u32x) (combs_buf[il_pos + 0].pw_len, combs_buf[il_pos + 1].pw_len, combs_buf[il_pos + 2].pw_len, combs_buf[il_pos + 3].pw_len, combs_buf[il_pos + 4].pw_len, combs_buf[il_pos + 5].pw_len, combs_buf[il_pos + 6].pw_len, combs_buf[il_pos + 7].pw_len, combs_buf[il_pos + 8].pw_len, combs_buf[il_pos + 9].pw_len, combs_buf[il_pos + 10].pw_len, combs_buf[il_pos + 11].pw_len, combs_buf[il_pos + 12].pw_len, combs_buf[il_pos + 13].pw_len, combs_buf[il_pos + 14].pw_len, combs_buf[il_pos + 15].pw_len);
-  #endif
-
-  return pw_lenx;
-}
-
-inline u32x ix_create_combt (__global comb_t *combs_buf, const u32 il_pos, const int idx)
-{
-  #if   VECT_SIZE == 1
-  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx]);
-  #elif VECT_SIZE == 2
-  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx]);
-  #elif VECT_SIZE == 4
-  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx]);
-  #elif VECT_SIZE == 8
-  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx], combs_buf[il_pos + 4].i[idx], combs_buf[il_pos + 5].i[idx], combs_buf[il_pos + 6].i[idx], combs_buf[il_pos + 7].i[idx]);
-  #elif VECT_SIZE == 16
-  const u32x ix = (u32x) (combs_buf[il_pos + 0].i[idx], combs_buf[il_pos + 1].i[idx], combs_buf[il_pos + 2].i[idx], combs_buf[il_pos + 3].i[idx], combs_buf[il_pos + 4].i[idx], combs_buf[il_pos + 5].i[idx], combs_buf[il_pos + 6].i[idx], combs_buf[il_pos + 7].i[idx], combs_buf[il_pos + 8].i[idx], combs_buf[il_pos + 9].i[idx], combs_buf[il_pos + 10].i[idx], combs_buf[il_pos + 11].i[idx], combs_buf[il_pos + 12].i[idx], combs_buf[il_pos + 13].i[idx], combs_buf[il_pos + 14].i[idx], combs_buf[il_pos + 15].i[idx]);
-  #endif
-
-  return ix;
-}
-
-#if   VECT_SIZE == 1
-#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) *  1) + 0].var[(idx)])
-#elif VECT_SIZE == 2
-#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) *  2) + 0].var[(idx)], (arr)[((gid) *  2) + 1].var[(idx)])
-#elif VECT_SIZE == 4
-#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) *  4) + 0].var[(idx)], (arr)[((gid) *  4) + 1].var[(idx)], (arr)[((gid) *  4) + 2].var[(idx)], (arr)[((gid) *  4) + 3].var[(idx)])
-#elif VECT_SIZE == 8
-#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) *  8) + 0].var[(idx)], (arr)[((gid) *  8) + 1].var[(idx)], (arr)[((gid) *  8) + 2].var[(idx)], (arr)[((gid) *  8) + 3].var[(idx)], (arr)[((gid) *  8) + 4].var[(idx)], (arr)[((gid) *  8) + 5].var[(idx)], (arr)[((gid) *  8) + 6].var[(idx)], (arr)[((gid) *  8) + 7].var[(idx)])
-#elif VECT_SIZE == 16
-#define packv(arr,var,gid,idx) (u32x) ((arr)[((gid) * 16) + 0].var[(idx)], (arr)[((gid) * 16) + 1].var[(idx)], (arr)[((gid) * 16) + 2].var[(idx)], (arr)[((gid) * 16) + 3].var[(idx)], (arr)[((gid) * 16) + 4].var[(idx)], (arr)[((gid) * 16) + 5].var[(idx)], (arr)[((gid) * 16) + 6].var[(idx)], (arr)[((gid) * 16) + 7].var[(idx)], (arr)[((gid) * 16) + 8].var[(idx)], (arr)[((gid) * 16) + 9].var[(idx)], (arr)[((gid) * 16) + 10].var[(idx)], (arr)[((gid) * 16) + 11].var[(idx)], (arr)[((gid) * 16) + 12].var[(idx)], (arr)[((gid) * 16) + 13].var[(idx)], (arr)[((gid) * 16) + 14].var[(idx)], (arr)[((gid) * 16) + 15].var[(idx)])
-#endif
-
-#if   VECT_SIZE == 1
-#define packvf(arr,var,gid) (u32x) ((arr)[((gid) *  1) + 0].var)
-#elif VECT_SIZE == 2
-#define packvf(arr,var,gid) (u32x) ((arr)[((gid) *  2) + 0].var, (arr)[((gid) *  2) + 1].var)
-#elif VECT_SIZE == 4
-#define packvf(arr,var,gid) (u32x) ((arr)[((gid) *  4) + 0].var, (arr)[((gid) *  4) + 1].var, (arr)[((gid) *  4) + 2].var, (arr)[((gid) *  4) + 3].var)
-#elif VECT_SIZE == 8
-#define packvf(arr,var,gid) (u32x) ((arr)[((gid) *  8) + 0].var, (arr)[((gid) *  8) + 1].var, (arr)[((gid) *  8) + 2].var, (arr)[((gid) *  8) + 3].var, (arr)[((gid) *  8) + 4].var, (arr)[((gid) *  8) + 5].var, (arr)[((gid) *  8) + 6].var, (arr)[((gid) *  8) + 7].var)
-#elif VECT_SIZE == 16
-#define packvf(arr,var,gid) (u32x) ((arr)[((gid) * 16) + 0].var, (arr)[((gid) * 16) + 1].var, (arr)[((gid) * 16) + 2].var, (arr)[((gid) * 16) + 3].var, (arr)[((gid) * 16) + 4].var, (arr)[((gid) * 16) + 5].var, (arr)[((gid) * 16) + 6].var, (arr)[((gid) * 16) + 7].var, (arr)[((gid) * 16) + 8].var, (arr)[((gid) * 16) + 9].var, (arr)[((gid) * 16) + 10].var, (arr)[((gid) * 16) + 11].var, (arr)[((gid) * 16) + 12].var, (arr)[((gid) * 16) + 13].var, (arr)[((gid) * 16) + 14].var, (arr)[((gid) * 16) + 15].var)
-#endif
-
-#if   VECT_SIZE == 1
-#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) *  1) + 0].var[(idx)])
-#elif VECT_SIZE == 2
-#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) *  2) + 0].var[(idx)], (arr)[((gid) *  2) + 1].var[(idx)])
-#elif VECT_SIZE == 4
-#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) *  4) + 0].var[(idx)], (arr)[((gid) *  4) + 1].var[(idx)], (arr)[((gid) *  4) + 2].var[(idx)], (arr)[((gid) *  4) + 3].var[(idx)])
-#elif VECT_SIZE == 8
-#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) *  8) + 0].var[(idx)], (arr)[((gid) *  8) + 1].var[(idx)], (arr)[((gid) *  8) + 2].var[(idx)], (arr)[((gid) *  8) + 3].var[(idx)], (arr)[((gid) *  8) + 4].var[(idx)], (arr)[((gid) *  8) + 5].var[(idx)], (arr)[((gid) *  8) + 6].var[(idx)], (arr)[((gid) *  8) + 7].var[(idx)])
-#elif VECT_SIZE == 16
-#define pack64v(arr,var,gid,idx) (u64x) ((arr)[((gid) * 16) + 0].var[(idx)], (arr)[((gid) * 16) + 1].var[(idx)], (arr)[((gid) * 16) + 2].var[(idx)], (arr)[((gid) * 16) + 3].var[(idx)], (arr)[((gid) * 16) + 4].var[(idx)], (arr)[((gid) * 16) + 5].var[(idx)], (arr)[((gid) * 16) + 6].var[(idx)], (arr)[((gid) * 16) + 7].var[(idx)], (arr)[((gid) * 16) + 8].var[(idx)], (arr)[((gid) * 16) + 9].var[(idx)], (arr)[((gid) * 16) + 10].var[(idx)], (arr)[((gid) * 16) + 11].var[(idx)], (arr)[((gid) * 16) + 12].var[(idx)], (arr)[((gid) * 16) + 13].var[(idx)], (arr)[((gid) * 16) + 14].var[(idx)], (arr)[((gid) * 16) + 15].var[(idx)])
-#endif
-
-#if   VECT_SIZE == 1
-#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) *  1) + 0].var)
-#elif VECT_SIZE == 2
-#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) *  2) + 0].var, (arr)[((gid) *  2) + 1].var)
-#elif VECT_SIZE == 4
-#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) *  4) + 0].var, (arr)[((gid) *  4) + 1].var, (arr)[((gid) *  4) + 2].var, (arr)[((gid) *  4) + 3].var)
-#elif VECT_SIZE == 8
-#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) *  8) + 0].var, (arr)[((gid) *  8) + 1].var, (arr)[((gid) *  8) + 2].var, (arr)[((gid) *  8) + 3].var, (arr)[((gid) *  8) + 4].var, (arr)[((gid) *  8) + 5].var, (arr)[((gid) *  8) + 6].var, (arr)[((gid) *  8) + 7].var)
-#elif VECT_SIZE == 16
-#define pack64vf(arr,var,gid) (u64x) ((arr)[((gid) * 16) + 0].var, (arr)[((gid) * 16) + 1].var, (arr)[((gid) * 16) + 2].var, (arr)[((gid) * 16) + 3].var, (arr)[((gid) * 16) + 4].var, (arr)[((gid) * 16) + 5].var, (arr)[((gid) * 16) + 6].var, (arr)[((gid) * 16) + 7].var, (arr)[((gid) * 16) + 8].var, (arr)[((gid) * 16) + 9].var, (arr)[((gid) * 16) + 10].var, (arr)[((gid) * 16) + 11].var, (arr)[((gid) * 16) + 12].var, (arr)[((gid) * 16) + 13].var, (arr)[((gid) * 16) + 14].var, (arr)[((gid) * 16) + 15].var)
-#endif
-
-#if   VECT_SIZE == 1
-#define unpackv(arr,var,gid,idx,val) (arr)[((gid) *  1) + 0].var[(idx)] = val;
-#elif VECT_SIZE == 2
-#define unpackv(arr,var,gid,idx,val) (arr)[((gid) *  2) + 0].var[(idx)] = val.s0; (arr)[((gid) *  2) + 1].var[(idx)] = val.s1;
-#elif VECT_SIZE == 4
-#define unpackv(arr,var,gid,idx,val) (arr)[((gid) *  4) + 0].var[(idx)] = val.s0; (arr)[((gid) *  4) + 1].var[(idx)] = val.s1; (arr)[((gid) *  4) + 2].var[(idx)] = val.s2; (arr)[((gid) *  4) + 3].var[(idx)] = val.s3;
-#elif VECT_SIZE == 8
-#define unpackv(arr,var,gid,idx,val) (arr)[((gid) *  8) + 0].var[(idx)] = val.s0; (arr)[((gid) *  8) + 1].var[(idx)] = val.s1; (arr)[((gid) *  8) + 2].var[(idx)] = val.s2; (arr)[((gid) *  8) + 3].var[(idx)] = val.s3; (arr)[((gid) *  8) + 4].var[(idx)] = val.s4; (arr)[((gid) *  8) + 5].var[(idx)] = val.s5; (arr)[((gid) *  8) + 6].var[(idx)] = val.s6; (arr)[((gid) *  8) + 7].var[(idx)] = val.s7;
-#elif VECT_SIZE == 16
-#define unpackv(arr,var,gid,idx,val) (arr)[((gid) * 16) + 0].var[(idx)] = val.s0; (arr)[((gid) * 16) + 1].var[(idx)] = val.s1; (arr)[((gid) * 16) + 2].var[(idx)] = val.s2; (arr)[((gid) * 16) + 3].var[(idx)] = val.s3; (arr)[((gid) * 16) + 4].var[(idx)] = val.s4; (arr)[((gid) * 16) + 5].var[(idx)] = val.s5; (arr)[((gid) * 16) + 6].var[(idx)] = val.s6; (arr)[((gid) * 16) + 7].var[(idx)] = val.s7; (arr)[((gid) * 16) + 8].var[(idx)] = val.s8; (arr)[((gid) * 16) + 9].var[(idx)] = val.s9; (arr)[((gid) * 16) + 10].var[(idx)] = val.sa; (arr)[((gid) * 16) + 11].var[(idx)] = val.sb; (arr)[((gid) * 16) + 12].var[(idx)] = val.sc; (arr)[((gid) * 16) + 13].var[(idx)] = val.sd; (arr)[((gid) * 16) + 14].var[(idx)] = val.se; (arr)[((gid) * 16) + 15].var[(idx)] = val.sf;
-#endif

diff --git a/OpenCL/types_ocl.c b/OpenCL/types_ocl.c
deleted file mode 100644 (file)
index 1d3a7b8..0000000
--- a/OpenCL/types_ocl.c
+++ /dev/null
@@ -1,1480 +0,0 @@
-/**
- * Authors.....: Jens Steube <jens.steube@gmail.com>
- *               magnum <john.magnum@hushmail.com>
- *
- * License.....: MIT
- */
-
-typedef uchar  u8;
-typedef ushort u16;
-typedef uint   u32;
-typedef ulong  u64;
-
-#ifndef NEW_SIMD_CODE
-#undef  VECT_SIZE
-#define VECT_SIZE 1
-#endif
-
-#define CONCAT(a, b)       a##b
-#define VTYPE(type, width) CONCAT(type, width)
-
-#if VECT_SIZE == 1
-typedef uchar   u8x;
-typedef ushort  u16x;
-typedef uint    u32x;
-typedef ulong   u64x;
-#else
-typedef VTYPE(uchar,  VECT_SIZE)  u8x;
-typedef VTYPE(ushort, VECT_SIZE) u16x;
-typedef VTYPE(uint,   VECT_SIZE) u32x;
-typedef VTYPE(ulong,  VECT_SIZE) u64x;
-#endif
-
-inline u32 l32_from_64_S (u64 a)
-{
-  const u32 r = (u32) (a);
-
-  return r;
-}
-
-inline u32 h32_from_64_S (u64 a)
-{
-  a >>= 32;
-
-  const u32 r = (u32) (a);
-
-  return r;
-}
-
-inline u64 hl32_to_64_S (const u32 a, const u32 b)
-{
-  return as_ulong ((uint2) (b, a));
-}
-
-inline u32x l32_from_64 (u64x a)
-{
-  u32x r;
-
-  #if VECT_SIZE == 1
-  r    = (u32) a;
-  #endif
-
-  #if VECT_SIZE >= 2
-  r.s0 = (u32) a.s0;
-  r.s1 = (u32) a.s1;
-  #endif
-
-  #if VECT_SIZE >= 4
-  r.s2 = (u32) a.s2;
-  r.s3 = (u32) a.s3;
-  #endif
-
-  #if VECT_SIZE >= 8
-  r.s4 = (u32) a.s4;
-  r.s5 = (u32) a.s5;
-  r.s6 = (u32) a.s6;
-  r.s7 = (u32) a.s7;
-  #endif
-
-  #if VECT_SIZE >= 16
-  r.s8 = (u32) a.s8;
-  r.s9 = (u32) a.s9;
-  r.sa = (u32) a.sa;
-  r.sb = (u32) a.sb;
-  r.sc = (u32) a.sc;
-  r.sd = (u32) a.sd;
-  r.se = (u32) a.se;
-  r.sf = (u32) a.sf;
-  #endif
-
-  return r;
-}
-
-inline u32x h32_from_64 (u64x a)
-{
-  a >>= 32;
-
-  u32x r;
-
-  #if VECT_SIZE == 1
-  r    = (u32) a;
-  #endif
-
-  #if VECT_SIZE >= 2
-  r.s0 = (u32) a.s0;
-  r.s1 = (u32) a.s1;
-  #endif
-
-  #if VECT_SIZE >= 4
-  r.s2 = (u32) a.s2;
-  r.s3 = (u32) a.s3;
-  #endif
-
-  #if VECT_SIZE >= 8
-  r.s4 = (u32) a.s4;
-  r.s5 = (u32) a.s5;
-  r.s6 = (u32) a.s6;
-  r.s7 = (u32) a.s7;
-  #endif
-
-  #if VECT_SIZE >= 16
-  r.s8 = (u32) a.s8;
-  r.s9 = (u32) a.s9;
-  r.sa = (u32) a.sa;
-  r.sb = (u32) a.sb;
-  r.sc = (u32) a.sc;
-  r.sd = (u32) a.sd;
-  r.se = (u32) a.se;
-  r.sf = (u32) a.sf;
-  #endif
-
-  return r;
-}
-
-inline u64x hl32_to_64 (const u32x a, const u32x b)
-{
-  u64x r;
-
-  #if VECT_SIZE == 1
-  r    = as_ulong ((uint2) (b,    a));
-  #endif
-
-  #if VECT_SIZE >= 2
-  r.s0 = as_ulong ((uint2) (b.s0, a.s0));
-  r.s1 = as_ulong ((uint2) (b.s1, a.s1));
-  #endif
-
-  #if VECT_SIZE >= 4
-  r.s2 = as_ulong ((uint2) (b.s2, a.s2));
-  r.s3 = as_ulong ((uint2) (b.s3, a.s3));
-  #endif
-
-  #if VECT_SIZE >= 8
-  r.s4 = as_ulong ((uint2) (b.s4, a.s4));
-  r.s5 = as_ulong ((uint2) (b.s5, a.s5));
-  r.s6 = as_ulong ((uint2) (b.s6, a.s6));
-  r.s7 = as_ulong ((uint2) (b.s7, a.s7));
-  #endif
-
-  #if VECT_SIZE >= 16
-  r.s8 = as_ulong ((uint2) (b.s8, a.s8));
-  r.s9 = as_ulong ((uint2) (b.s9, a.s9));
-  r.sa = as_ulong ((uint2) (b.sa, a.sa));
-  r.sb = as_ulong ((uint2) (b.sb, a.sb));
-  r.sc = as_ulong ((uint2) (b.sc, a.sc));
-  r.sd = as_ulong ((uint2) (b.sd, a.sd));
-  r.se = as_ulong ((uint2) (b.se, a.se));
-  r.sf = as_ulong ((uint2) (b.sf, a.sf));
-  #endif
-
-  return r;
-}
-
-#ifdef IS_AMD
-inline u32 swap32_S (const u32 v)
-{
-  return (as_uint (as_uchar4 (v).s3210));
-}
-
-inline u64 swap64_S (const u64 v)
-{
-  return (as_ulong (as_uchar8 (v).s76543210));
-}
-
-inline u32 rotr32_S (const u32 a, const u32 n)
-{
-  return rotate (a, 32 - n);
-}
-
-inline u32 rotl32_S (const u32 a, const u32 n)
-{
-  return rotate (a, n);
-}
-
-inline u64 rotr64_S (const u64 a, const u32 n)
-{
-  const u32 a0 = h32_from_64_S (a);
-  const u32 a1 = l32_from_64_S (a);
-
-  const u32 t0 = (n >= 32) ? amd_bitalign (a0, a1, n - 32) : amd_bitalign (a1, a0, n);
-  const u32 t1 = (n >= 32) ? amd_bitalign (a1, a0, n - 32) : amd_bitalign (a0, a1, n);
-
-  const u64 r = hl32_to_64_S (t0, t1);
-
-  return r;
-}
-
-inline u64 rotl64_S (const u64 a, const u32 n)
-{
-  return rotr64_S (a, 64 - n);
-}
-
-inline u32x swap32 (const u32x v)
-{
-  return ((v >> 24) & 0x000000ff)
-       | ((v >>  8) & 0x0000ff00)
-       | ((v <<  8) & 0x00ff0000)
-       | ((v << 24) & 0xff000000);
-}
-
-inline u64x swap64 (const u64x v)
-{
-  return ((v >> 56) & 0x00000000000000ff)
-       | ((v >> 40) & 0x000000000000ff00)
-       | ((v >> 24) & 0x0000000000ff0000)
-       | ((v >>  8) & 0x00000000ff000000)
-       | ((v <<  8) & 0x000000ff00000000)
-       | ((v << 24) & 0x0000ff0000000000)
-       | ((v << 40) & 0x00ff000000000000)
-       | ((v << 56) & 0xff00000000000000);
-}
-
-inline u32x rotr32 (const u32x a, const u32 n)
-{
-  return rotate (a, 32 - n);
-}
-
-inline u32x rotl32 (const u32x a, const u32 n)
-{
-  return rotate (a, n);
-}
-
-inline u64x rotr64 (const u64x a, const u32 n)
-{
-  const u32x a0 = h32_from_64 (a);
-  const u32x a1 = l32_from_64 (a);
-
-  const u32x t0 = (n >= 32) ? amd_bitalign (a0, a1, n - 32) : amd_bitalign (a1, a0, n);
-  const u32x t1 = (n >= 32) ? amd_bitalign (a1, a0, n - 32) : amd_bitalign (a0, a1, n);
-
-  const u64x r = hl32_to_64 (t0, t1);
-
-  return r;
-}
-
-inline u64x rotl64 (const u64x a, const u32 n)
-{
-  return rotr64 (a, 64 - n);
-}
-
-inline u32x __bfe (const u32x a, const u32x b, const u32x c)
-{
-  return amd_bfe (a, b, c);
-}
-
-inline u32 __bfe_S (const u32 a, const u32 b, const u32 c)
-{
-  return amd_bfe (a, b, c);
-}
-
-inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c)
-{
-  return amd_bytealign (a, b, c);
-}
-#endif
-
-#ifdef IS_NV
-inline u32 swap32_S (const u32 v)
-{
-  u32 r;
-
-  asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(r) : "r"(v));
-
-  return r;
-}
-
-inline u64 swap64_S (const u64 v)
-{
-  u32 il;
-  u32 ir;
-
-  asm ("mov.b64 {%0, %1}, %2;" : "=r"(il), "=r"(ir) : "l"(v));
-
-  u32 tl;
-  u32 tr;
-
-  asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(tl) : "r"(il));
-  asm ("prmt.b32 %0, %1, 0, 0x0123;" : "=r"(tr) : "r"(ir));
-
-  u64 r;
-
-  asm ("mov.b64 %0, {%1, %2};" : "=l"(r) : "r"(tr), "r"(tl));
-
-  return r;
-}
-
-inline u32 rotr32_S (const u32 a, const u32 n)
-{
-  return rotate (a, 32 - n);
-}
-
-inline u32 rotl32_S (const u32 a, const u32 n)
-{
-  return rotate (a, n);
-}
-
-inline u64 rotr64_S (const u64 a, const u32 n)
-{
-  return rotate (a, (u64) 64 - n);
-}
-
-inline u64 rotl64_S (const u64 a, const u32 n)
-{
-  return rotr64_S (a, 64 - n);
-}
-
-inline u32x swap32 (const u32x v)
-{
-  return ((v >> 24) & 0x000000ff)
-       | ((v >>  8) & 0x0000ff00)
-       | ((v <<  8) & 0x00ff0000)
-       | ((v << 24) & 0xff000000);
-}
-
-inline u64x swap64 (const u64x v)
-{
-  return ((v >> 56) & 0x00000000000000ff)
-       | ((v >> 40) & 0x000000000000ff00)
-       | ((v >> 24) & 0x0000000000ff0000)
-       | ((v >>  8) & 0x00000000ff000000)
-       | ((v <<  8) & 0x000000ff00000000)
-       | ((v << 24) & 0x0000ff0000000000)
-       | ((v << 40) & 0x00ff000000000000)
-       | ((v << 56) & 0xff00000000000000);
-}
-
-inline u32x rotr32 (const u32x a, const u32 n)
-{
-  return rotate (a, 32 - n);
-}
-
-inline u32x rotl32 (const u32x a, const u32 n)
-{
-  return rotate (a, n);
-}
-
-inline u64x rotr64 (const u64x a, const u32 n)
-{
-  return rotate (a, (u64) 64 - n);
-}
-
-inline u64x rotl64 (const u64x a, const u32 n)
-{
-  return rotate (a, (u64) n);
-}
-
-inline u32x __byte_perm (const u32x a, const u32x b, const u32x c)
-{
-  u32x r;
-
-  #if VECT_SIZE == 1
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r)    : "r"(a),    "r"(b),    "r"(c)   );
-  #endif
-
-  #if VECT_SIZE >= 2
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(a.s0), "r"(b.s0), "r"(c.s0));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(a.s1), "r"(b.s1), "r"(c.s1));
-  #endif
-
-  #if VECT_SIZE >= 4
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(a.s2), "r"(b.s2), "r"(c.s2));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(a.s3), "r"(b.s3), "r"(c.s3));
-  #endif
-
-  #if VECT_SIZE >= 8
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(a.s4), "r"(b.s4), "r"(c.s4));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(a.s5), "r"(b.s5), "r"(c.s5));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(a.s6), "r"(b.s6), "r"(c.s6));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(a.s7), "r"(b.s7), "r"(c.s7));
-  #endif
-
-  #if VECT_SIZE >= 16
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(a.s8), "r"(b.s8), "r"(c.s8));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(a.s9), "r"(b.s9), "r"(c.s9));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(a.sa), "r"(b.sa), "r"(c.sa));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(a.sb), "r"(b.sb), "r"(c.sb));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(a.sc), "r"(b.sc), "r"(c.sc));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(a.sd), "r"(b.sd), "r"(c.sd));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(a.se), "r"(b.se), "r"(c.se));
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(a.sf), "r"(b.sf), "r"(c.sf));
-  #endif
-
-  return r;
-}
-
-inline u32 __byte_perm_S (const u32 a, const u32 b, const u32 c)
-{
-  u32 r;
-
-  asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c));
-
-  return r;
-}
-
-inline u32x __bfe (const u32x a, const u32x b, const u32x c)
-{
-  u32x r;
-
-  #if VECT_SIZE == 1
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r)    : "r"(a),    "r"(b),    "r"(c));
-  #endif
-
-  #if VECT_SIZE >= 2
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(a.s0), "r"(b.s0), "r"(c.s0));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(a.s1), "r"(b.s1), "r"(c.s1));
-  #endif
-
-  #if VECT_SIZE >= 4
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(a.s2), "r"(b.s2), "r"(c.s2));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(a.s3), "r"(b.s3), "r"(c.s3));
-  #endif
-
-  #if VECT_SIZE >= 8
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(a.s4), "r"(b.s4), "r"(c.s4));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(a.s5), "r"(b.s5), "r"(c.s5));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(a.s6), "r"(b.s6), "r"(c.s6));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(a.s7), "r"(b.s7), "r"(c.s7));
-  #endif
-
-  #if VECT_SIZE >= 16
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(a.s8), "r"(b.s8), "r"(c.s8));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(a.s9), "r"(b.s9), "r"(c.s9));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(a.sa), "r"(b.sa), "r"(c.sa));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(a.sb), "r"(b.sb), "r"(c.sb));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(a.sc), "r"(b.sc), "r"(c.sc));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(a.sd), "r"(b.sd), "r"(c.sd));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(a.se), "r"(b.se), "r"(c.se));
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(a.sf), "r"(b.sf), "r"(c.sf));
-  #endif
-
-  return r;
-}
-
-inline u32 __bfe_S (const u32 a, const u32 b, const u32 c)
-{
-  u32 r;
-
-  asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c));
-
-  return r;
-}
-
-inline u32x amd_bytealign (const u32x a, const u32x b, const u32x c)
-{
-  u32x r;
-
-  #if CUDA_ARCH >= 350
-
-  #if VECT_SIZE == 1
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r)    : "r"(b),    "r"(a),    "r"((c & 3) * 8));
-  #endif
-
-  #if VECT_SIZE >= 2
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s0) : "r"(b.s0), "r"(a.s0), "r"((c.s0 & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s1) : "r"(b.s1), "r"(a.s1), "r"((c.s1 & 3) * 8));
-  #endif
-
-  #if VECT_SIZE >= 4
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s2) : "r"(b.s2), "r"(a.s2), "r"((c.s2 & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s3) : "r"(b.s3), "r"(a.s3), "r"((c.s3 & 3) * 8));
-  #endif
-
-  #if VECT_SIZE >= 8
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s4) : "r"(b.s4), "r"(a.s4), "r"((c.s4 & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s5) : "r"(b.s5), "r"(a.s5), "r"((c.s5 & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s6) : "r"(b.s6), "r"(a.s6), "r"((c.s6 & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s7) : "r"(b.s7), "r"(a.s7), "r"((c.s7 & 3) * 8));
-  #endif
-
-  #if VECT_SIZE >= 16
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s8) : "r"(b.s8), "r"(a.s8), "r"((c.s8 & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.s9) : "r"(b.s9), "r"(a.s9), "r"((c.s9 & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sa) : "r"(b.sa), "r"(a.sa), "r"((c.sa & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sb) : "r"(b.sb), "r"(a.sb), "r"((c.sb & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sc) : "r"(b.sc), "r"(a.sc), "r"((c.sc & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sd) : "r"(b.sd), "r"(a.sd), "r"((c.sd & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.se) : "r"(b.se), "r"(a.se), "r"((c.se & 3) * 8));
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r.sf) : "r"(b.sf), "r"(a.sf), "r"((c.sf & 3) * 8));
-  #endif
-
-  #else
-
-  r = __byte_perm (b, a, ((u32x) (0x76543210) >> ((c & 3) * 4)) & 0xffff);
-
-  #endif
-
-  return r;
-}
-
-inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c)
-{
-  u32 r;
-
-  #if CUDA_ARCH >= 350
-
-  asm ("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(r) : "r"(b), "r"(a), "r"((c & 3) * 8));
-
-  #else
-
-  r = __byte_perm_S (b, a, (0x76543210 >> ((c & 3) * 4)) & 0xffff);
-
-  #endif
-
-  return r;
-}
-#endif
-
-#ifdef IS_GENERIC
-inline u32 swap32_S (const u32 v)
-{
-  return (as_uint (as_uchar4 (v).s3210));
-}
-
-inline u64 swap64_S (const u64 v)
-{
-  return (as_ulong (as_uchar8 (v).s76543210));
-}
-
-inline u32 rotr32_S (const u32 a, const u32 n)
-{
-  return rotate (a, 32 - n);
-}
-
-inline u32 rotl32_S (const u32 a, const u32 n)
-{
-  return rotate (a, n);
-}
-
-inline u64 rotr64_S (const u64 a, const u32 n)
-{
-  return rotate (a, (u64) 64 - n);
-}
-
-inline u64 rotl64_S (const u64 a, const u32 n)
-{
-  return rotate (a, (u64) n);
-}
-
-inline u32x swap32 (const u32x v)
-{
-  return ((v >> 24) & 0x000000ff)
-       | ((v >>  8) & 0x0000ff00)
-       | ((v <<  8) & 0x00ff0000)
-       | ((v << 24) & 0xff000000);
-}
-
-inline u64x swap64 (const u64x v)
-{
-  return ((v >> 56) & 0x00000000000000ff)
-       | ((v >> 40) & 0x000000000000ff00)
-       | ((v >> 24) & 0x0000000000ff0000)
-       | ((v >>  8) & 0x00000000ff000000)
-       | ((v <<  8) & 0x000000ff00000000)
-       | ((v << 24) & 0x0000ff0000000000)
-       | ((v << 40) & 0x00ff000000000000)
-       | ((v << 56) & 0xff00000000000000);
-}
-
-inline u32x rotr32 (const u32x a, const u32 n)
-{
-  return rotate (a, 32 - n);
-}
-
-inline u32x rotl32 (const u32x a, const u32 n)
-{
-  return rotate (a, n);
-}
-
-inline u64x rotr64 (const u64x a, const u32 n)
-{
-  return rotate (a, (u64) 64 - n);
-}
-
-inline u64x rotl64 (const u64x a, const u32 n)
-{
-  return rotate (a, (u64) n);
-}
-
-inline u32x __bfe (const u32x a, const u32x b, const u32x c)
-{
-  #define BIT(x)      ((u32x) (1u) << (x))
-  #define BIT_MASK(x) (BIT (x) - 1)
-  #define BFE(x,y,z)  (((x) >> (y)) & BIT_MASK (z))
-
-  return BFE (a, b, c);
-
-  #undef BIT
-  #undef BIT_MASK
-  #undef BFE
-}
-
-inline u32 __bfe_S (const u32 a, const u32 b, const u32 c)
-{
-  #define BIT(x)      (1u << (x))
-  #define BIT_MASK(x) (BIT (x) - 1)
-  #define BFE(x,y,z)  (((x) >> (y)) & BIT_MASK (z))
-
-  return BFE (a, b, c);
-
-  #undef BIT
-  #undef BIT_MASK
-  #undef BFE
-}
-
-inline u32x amd_bytealign (const u32x a, const u32x b, const u32 c)
-{
-  #if VECT_SIZE == 1
-  const u64x tmp = ((((u64x) (a)) << 32) | ((u64x) (b))) >> ((c & 3) * 8);
-
-  return (u32x) (tmp);
-  #endif
-
-  #if VECT_SIZE == 2
-  const u64x tmp = ((((u64x) (a.s0, a.s1)) << 32) | ((u64x) (b.s0, b.s1))) >> ((c & 3) * 8);
-
-  return (u32x) (tmp.s0, tmp.s1);
-  #endif
-
-  #if VECT_SIZE == 4
-  const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3))) >> ((c & 3) * 8);
-
-  return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3);
-  #endif
-
-  #if VECT_SIZE == 8
-  const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3, a.s4, a.s5, a.s6, a.s7)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3, b.s4, b.s5, b.s6, b.s7))) >> ((c & 3) * 8);
-
-  return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3, tmp.s4, tmp.s5, tmp.s6, tmp.s7);
-  #endif
-
-  #if VECT_SIZE == 16
-  const u64x tmp = ((((u64x) (a.s0, a.s1, a.s2, a.s3, a.s4, a.s5, a.s6, a.s7, a.s8, a.s9, a.sa, a.sb, a.sc, a.sd, a.se, a.sf)) << 32) | ((u64x) (b.s0, b.s1, b.s2, b.s3, b.s4, b.s5, b.s6, b.s7, b.s8, b.s9, b.sa, b.sb, b.sc, b.sd, b.se, b.sf))) >> ((c & 3) * 8);
-
-  return (u32x) (tmp.s0, tmp.s1, tmp.s2, tmp.s3, tmp.s4, tmp.s5, tmp.s6, tmp.s7, tmp.s8, tmp.s9, tmp.sa, tmp.sb, tmp.sc, tmp.sd, tmp.se, tmp.sf);
-  #endif
-}
-
-inline u32 amd_bytealign_S (const u32 a, const u32 b, const u32 c)
-{
-  const u64 tmp = ((((u64) a) << 32) | ((u64) b)) >> ((c & 3) * 8);
-
-  return (u32) (tmp);
-}
-
-#endif
-
-typedef struct
-{
-  #if   defined _DES_
-  u32  digest_buf[4];
-  #elif defined _MD4_
-  u32  digest_buf[4];
-  #elif defined _MD5_
-  u32  digest_buf[4];
-  #elif defined _MD5H_
-  u32  digest_buf[4];
-  #elif defined _SHA1_
-  u32  digest_buf[5];
-  #elif defined _BCRYPT_
-  u32  digest_buf[6];
-  #elif defined _SHA256_
-  u32  digest_buf[8];
-  #elif defined _SHA384_
-  u32  digest_buf[16];
-  #elif defined _SHA512_
-  u32  digest_buf[16];
-  #elif defined _KECCAK_
-  u32  digest_buf[50];
-  #elif defined _RIPEMD160_
-  u32  digest_buf[5];
-  #elif defined _WHIRLPOOL_
-  u32  digest_buf[16];
-  #elif defined _GOST_
-  u32  digest_buf[8];
-  #elif defined _GOST2012_256_
-  u32  digest_buf[8];
-  #elif defined _GOST2012_512_
-  u32  digest_buf[16];
-  #elif defined _SAPB_
-  u32  digest_buf[4];
-  #elif defined _SAPG_
-  u32  digest_buf[5];
-  #elif defined _MYSQL323_
-  u32  digest_buf[4];
-  #elif defined _LOTUS5_
-  u32  digest_buf[4];
-  #elif defined _LOTUS6_
-  u32  digest_buf[4];
-  #elif defined _SCRYPT_
-  u32  digest_buf[8];
-  #elif defined _LOTUS8_
-  u32  digest_buf[4];
-  #elif defined _OFFICE2007_
-  u32  digest_buf[4];
-  #elif defined _OFFICE2010_
-  u32  digest_buf[4];
-  #elif defined _OFFICE2013_
-  u32  digest_buf[4];
-  #elif defined _OLDOFFICE01_
-  u32  digest_buf[4];
-  #elif defined _OLDOFFICE34_
-  u32  digest_buf[4];
-  #elif defined _SIPHASH_
-  u32  digest_buf[4];
-  #elif defined _PBKDF2_MD5_
-  u32  digest_buf[32];
-  #elif defined _PBKDF2_SHA1_
-  u32  digest_buf[32];
-  #elif defined _PBKDF2_SHA256_
-  u32  digest_buf[32];
-  #elif defined _PBKDF2_SHA512_
-  u32  digest_buf[32];
-  #elif defined _PDF17L8_
-  u32  digest_buf[8];
-  #elif defined _CRC32_
-  u32  digest_buf[4];
-  #elif defined _SEVEN_ZIP_
-  u32  digest_buf[4];
-  #elif defined _ANDROIDFDE_
-  u32  digest_buf[4];
-  #elif defined _DCC2_
-  u32  digest_buf[4];
-  #elif defined _WPA_
-  u32  digest_buf[4];
-  #elif defined _MD5_SHA1_
-  u32  digest_buf[4];
-  #elif defined _SHA1_MD5_
-  u32  digest_buf[5];
-  #elif defined _NETNTLMV2_
-  u32  digest_buf[4];
-  #elif defined _KRB5PA_
-  u32  digest_buf[4];
-  #elif defined _CLOUDKEY_
-  u32  digest_buf[8];
-  #elif defined _SCRYPT_
-  u32  digest_buf[4];
-  #elif defined _PSAFE2_
-  u32  digest_buf[5];
-  #elif defined _LOTUS8_
-  u32  digest_buf[4];
-  #elif defined _RAR3_
-  u32  digest_buf[4];
-  #elif defined _SHA256_SHA1_
-  u32  digest_buf[8];
-  #elif defined _MS_DRSR_
-  u32  digest_buf[8];
-  #elif defined _ANDROIDFDE_SAMSUNG_
-  u32  digest_buf[8];
-  #elif defined _RAR5_
-  u32  digest_buf[4];
-  #elif defined _KRB5TGS_
-  u32  digest_buf[4];
-  #elif defined _AXCRYPT_
-  u32  digest_buf[4];
-  #elif defined _KEEPASS_
-  u32  digest_buf[4];
-  #elif defined _ZIP2_
-  u32  digest_buf[4];
-  #endif
-
-} digest_t;
-
-typedef struct
-{
-  u32 salt_buf[16];
-  u32 salt_buf_pc[8];
-
-  u32 salt_len;
-  u32 salt_iter;
-  u32 salt_sign[2];
-
-  u32 keccak_mdlen;
-  u32 truecrypt_mdlen;
-
-  u32 digests_cnt;
-  u32 digests_done;
-
-  u32 digests_offset;
-
-  u32 scrypt_N;
-  u32 scrypt_r;
-  u32 scrypt_p;
-  u32 scrypt_tmto;
-  u32 scrypt_phy;
-
-} salt_t;
-
-typedef struct
-{
-  int V;
-  int R;
-  int P;
-
-  int enc_md;
-
-  u32 id_buf[8];
-  u32 u_buf[32];
-  u32 o_buf[32];
-
-  int id_len;
-  int o_len;
-  int u_len;
-
-  u32 rc4key[2];
-  u32 rc4data[2];
-
-} pdf_t;
-
-typedef struct
-{
-  u32 pke[25];
-  u32 eapol[64];
-  int eapol_size;
-  int keyver;
-  u8  orig_mac1[6];
-  u8  orig_mac2[6];
-  u8  orig_nonce1[32];
-  u8  orig_nonce2[32];
-
-} wpa_t;
-
-typedef struct
-{
-  u32 cry_master_buf[64];
-  u32 ckey_buf[64];
-  u32 public_key_buf[64];
-
-  u32 cry_master_len;
-  u32 ckey_len;
-  u32 public_key_len;
-
-} bitcoin_wallet_t;
-
-typedef struct
-{
-  u32 salt_buf[30];
-  u32 salt_len;
-
-  u32 esalt_buf[38];
-  u32 esalt_len;
-
-} sip_t;
-
-typedef struct
-{
-  u32 data[384];
-
-} androidfde_t;
-
-typedef struct
-{
-  u32 nr_buf[16];
-  u32 nr_len;
-
-  u32 msg_buf[128];
-  u32 msg_len;
-
-} ikepsk_t;
-
-typedef struct
-{
-  u32 user_len;
-  u32 domain_len;
-  u32 srvchall_len;
-  u32 clichall_len;
-
-  u32 userdomain_buf[64];
-  u32 chall_buf[256];
-
-} netntlm_t;
-
-typedef struct
-{
-  u32 user[16];
-  u32 realm[16];
-  u32 salt[32];
-  u32 timestamp[16];
-  u32 checksum[4];
-
-} krb5pa_t;
-
-typedef struct
-{
-  u32 account_info[512];
-  u32 checksum[4];
-  u32 edata2[2560];
-  u32 edata2_len;
-
-} krb5tgs_t;
-
-typedef struct
-{
-  u32 salt_buf[16];
-  u32 data_buf[112];
-  u32 keyfile_buf[16];
-  u32 signature;
-
-} tc_t;
-
-typedef struct
-{
-  u32 salt_buf[16];
-
-} pbkdf2_md5_t;
-
-typedef struct
-{
-  u32 salt_buf[16];
-
-} pbkdf2_sha1_t;
-
-typedef struct
-{
-  u32 salt_buf[16];
-
-} pbkdf2_sha256_t;
-
-typedef struct
-{
-  u32 salt_buf[32];
-
-} pbkdf2_sha512_t;
-
-typedef struct
-{
-  u32 salt_buf[128];
-  u32 salt_len;
-
-} rakp_t;
-
-typedef struct
-{
-  u32 data_len;
-  u32 data_buf[512];
-
-} cloudkey_t;
-
-typedef struct
-{
-  u32 encryptedVerifier[4];
-  u32 encryptedVerifierHash[5];
-
-  u32 keySize;
-
-} office2007_t;
-
-typedef struct
-{
-  u32 encryptedVerifier[4];
-  u32 encryptedVerifierHash[8];
-
-} office2010_t;
-
-typedef struct
-{
-  u32 encryptedVerifier[4];
-  u32 encryptedVerifierHash[8];
-
-} office2013_t;
-
-typedef struct
-{
-  u32 version;
-  u32 encryptedVerifier[4];
-  u32 encryptedVerifierHash[4];
-  u32 rc4key[2];
-
-} oldoffice01_t;
-
-typedef struct
-{
-  u32 version;
-  u32 encryptedVerifier[4];
-  u32 encryptedVerifierHash[5];
-  u32 rc4key[2];
-
-} oldoffice34_t;
-
-typedef struct
-{
-  u32 salt_buf[128];
-  u32 salt_len;
-
-  u32 pc_digest[5];
-  u32 pc_offset;
-
-} pstoken_t;
-
-typedef struct
-{
-  u32 type;
-  u32 mode;
-  u32 magic;
-  u32 salt_len;
-  u32 salt_buf[4];
-  u32 verify_bytes;
-  u32 compress_length;
-  u32 data_len;
-  u32 data_buf[2048];
-  u32 auth_len;
-  u32 auth_buf[4];
-
-} zip2_t;
-
-typedef struct
-{
-  u32 version;
-  u32 algorithm;
-
-  /* key-file handling */
-  u32 keyfile_len;
-  u32 keyfile[8];
-
-  u32 final_random_seed[8];
-  u32 transf_random_seed[8];
-  u32 enc_iv[4];
-  u32 contents_hash[8];
-
-  /* specific to version 1 */
-  u32 contents_len;
-  u32 contents[75000];
-
-  /* specific to version 2 */
-  u32 expected_bytes[8];
-
-} keepass_t;
-
-typedef struct
-{
-  u32 digest[4];
-  u32 out[4];
-
-} pdf14_tmp_t;
-
-typedef struct
-{
-  union
-  {
-    u32 dgst32[16];
-    u64 dgst64[8];
-  };
-
-  u32 dgst_len;
-  u32 W_len;
-
-} pdf17l8_tmp_t;
-
-typedef struct
-{
-  u32 digest_buf[4];
-
-} phpass_tmp_t;
-
-typedef struct
-{
-  u32 digest_buf[4];
-
-} md5crypt_tmp_t;
-
-typedef struct
-{
-  u32 alt_result[8];
-
-  u32 p_bytes[4];
-  u32 s_bytes[4];
-
-} sha256crypt_tmp_t;
-
-typedef struct
-{
-  u64 l_alt_result[8];
-
-  u64 l_p_bytes[2];
-  u64 l_s_bytes[2];
-
-} sha512crypt_tmp_t;
-
-typedef struct
-{
-  u32 ipad[5];
-  u32 opad[5];
-
-  u32 dgst[10];
-  u32 out[10];
-
-} wpa_tmp_t;
-
-typedef struct
-{
-  u64 dgst[8];
-
-} bitcoin_wallet_tmp_t;
-
-typedef struct
-{
-  u32 ipad[5];
-  u32 opad[5];
-
-  u32 dgst[5];
-  u32 out[4];
-
-} dcc2_tmp_t;
-
-typedef struct
-{
-  u32 E[18];
-
-  u32 P[18];
-
-  u32 S0[256];
-  u32 S1[256];
-  u32 S2[256];
-  u32 S3[256];
-
-} bcrypt_tmp_t;
-
-typedef struct
-{
-  u32 digest[2];
-
-  u32 P[18];
-
-  u32 S0[256];
-  u32 S1[256];
-  u32 S2[256];
-  u32 S3[256];
-
-} pwsafe2_tmp_t;
-
-typedef struct
-{
-  u32 digest_buf[8];
-
-} pwsafe3_tmp_t;
-
-typedef struct
-{
-  u32 digest_buf[5];
-
-} androidpin_tmp_t;
-
-typedef struct
-{
-  u32 ipad[5];
-  u32 opad[5];
-
-  u32 dgst[10];
-  u32 out[10];
-
-} androidfde_tmp_t;
-
-typedef struct
-{
-  u32 ipad[16];
-  u32 opad[16];
-
-  u32 dgst[64];
-  u32 out[64];
-
-} tc_tmp_t;
-
-typedef struct
-{
-  u64 ipad[8];
-  u64 opad[8];
-
-  u64 dgst[32];
-  u64 out[32];
-
-} tc64_tmp_t;
-
-typedef struct
-{
-  u32 ipad[4];
-  u32 opad[4];
-
-  u32 dgst[32];
-  u32 out[32];
-
-} pbkdf2_md5_tmp_t;
-
-typedef struct
-{
-  u32 ipad[5];
-  u32 opad[5];
-
-  u32 dgst[32];
-  u32 out[32];
-
-} pbkdf2_sha1_tmp_t;
-
-typedef struct
-{
-  u32 ipad[8];
-  u32 opad[8];
-
-  u32 dgst[32];
-  u32 out[32];
-
-} pbkdf2_sha256_tmp_t;
-
-typedef struct
-{
-  u64 ipad[8];
-  u64 opad[8];
-
-  u64 dgst[16];
-  u64 out[16];
-
-} pbkdf2_sha512_tmp_t;
-
-typedef struct
-{
-  u64 out[8];
-
-} ecryptfs_tmp_t;
-
-typedef struct
-{
-  u64 ipad[8];
-  u64 opad[8];
-
-  u64 dgst[16];
-  u64 out[16];
-
-} oraclet_tmp_t;
-
-typedef struct
-{
-  u32 ipad[5];
-  u32 opad[5];
-
-  u32 dgst[5];
-  u32 out[5];
-
-} agilekey_tmp_t;
-
-typedef struct
-{
-  u32 ipad[5];
-  u32 opad[5];
-
-  u32 dgst1[5];
-  u32 out1[5];
-
-  u32 dgst2[5];
-  u32 out2[5];
-
-} mywallet_tmp_t;
-
-typedef struct
-{
-  u32 ipad[5];
-  u32 opad[5];
-
-  u32 dgst[5];
-  u32 out[5];
-
-} sha1aix_tmp_t;
-
-typedef struct
-{
-  u32 ipad[8];
-  u32 opad[8];
-
-  u32 dgst[8];
-  u32 out[8];
-
-} sha256aix_tmp_t;
-
-typedef struct
-{
-  u64 ipad[8];
-  u64 opad[8];
-
-  u64 dgst[8];
-  u64 out[8];
-
-} sha512aix_tmp_t;
-
-typedef struct
-{
-  u32 ipad[8];
-  u32 opad[8];
-
-  u32 dgst[8];
-  u32 out[8];
-
-} lastpass_tmp_t;
-
-typedef struct
-{
-  u64 digest_buf[8];
-
-} drupal7_tmp_t;
-
-typedef struct
-{
-  u32 ipad[5];
-  u32 opad[5];
-
-  u32 dgst[5];
-  u32 out[5];
-
-} lotus8_tmp_t;
-
-typedef struct
-{
-  u32 out[5];
-
-} office2007_tmp_t;
-
-typedef struct
-{
-  u32 out[5];
-
-} office2010_tmp_t;
-
-typedef struct
-{
-  u64 out[8];
-
-} office2013_tmp_t;
-
-typedef struct
-{
-  u32 digest_buf[5];
-
-} saph_sha1_tmp_t;
-
-typedef struct
-{
-  u32 block[16];
-
-  u32 dgst[8];
-
-  u32 block_len;
-  u32 final_len;
-
-} seven_zip_tmp_t;
-
-typedef struct
-{
-  u32 KEK[5];
-
-  u32 lsb[4];
-  u32 cipher[4];
-
-} axcrypt_tmp_t;
-
-typedef struct
-{
-  u32 tmp_digest[8];
-
-} keepass_tmp_t;
-
-typedef struct
-{
-  u32 Kc[16];
-  u32 Kd[16];
-
-  u32 iv[2];
-
-} bsdicrypt_tmp_t;
-
-typedef struct
-{
-  u32 dgst[17][5];
-
-} rar3_tmp_t;
-
-typedef struct
-{
-  u32 user[16];
-
-} cram_md5_t;
-
-typedef struct
-{
-  u32 iv_buf[4];
-  u32 iv_len;
-
-  u32 salt_buf[4];
-  u32 salt_len;
-
-  u32 crc;
-
-  u32 data_buf[96];
-  u32 data_len;
-
-  u32 unpack_size;
-
-} seven_zip_t;
-
-typedef struct
-{
-  u32 key;
-  u64 val;
-
-} hcstat_table_t;
-
-typedef struct
-{
-  u32 cs_buf[0x100];
-  u32 cs_len;
-
-} cs_t;
-
-typedef struct
-{
-  u32 cmds[0x100];
-
-} kernel_rule_t;
-
-typedef struct
-{
-  u32 salt_pos;
-  u32 digest_pos;
-  u32 hash_pos;
-  u32 gidvid;
-  u32 il_pos;
-
-} plain_t;
-
-typedef struct
-{
-  u32 i[16];
-
-  u32 pw_len;
-
-  u32 alignment_placeholder_1;
-  u32 alignment_placeholder_2;
-  u32 alignment_placeholder_3;
-
-} pw_t;
-
-typedef struct
-{
-  u32 i;
-
-} bf_t;
-
-typedef struct
-{
-  u32 i[8];
-
-  u32 pw_len;
-
-} comb_t;
-
-typedef struct
-{
-  u32 b[32];
-
-} bs_word_t;
-
-typedef struct
-{
-  uint4 P[64];
-
-} scrypt_tmp_t;

diff --git a/include/constants.h b/include/constants.h
deleted file mode 100644 (file)
index 5351445..0000000
--- a/include/constants.h
+++ /dev/null
@@ -1,758 +0,0 @@
-/**
- * Author......: Jens Steube <jens.steube@gmail.com>
- * License.....: MIT
- */
-
-#define COMBINATOR_MODE_BASE_LEFT  10001
-#define COMBINATOR_MODE_BASE_RIGHT 10002
-
-#ifdef SHARED_H
-#define _BCRYPT_
-#define _SHA1_
-#define _SHA256_
-#define _SHA384_
-#define _SHA512_
-#define _MD4_
-#define _MD5_
-#define _MD5H_
-#define _KECCAK_
-#define _RIPEMD160_
-#define _WHIRLPOOL_
-#define _SAPB_
-#define _CLOUDKEY_
-#define _OFFICE2013_
-#define _SIPHASH_
-#define _PDF17L8_
-#define _PBKDF2_MD5_
-#define _PBKDF2_SHA1_
-#define _PBKDF2_SHA256_
-#define _PBKDF2_SHA512_
-#define _RAR3_
-#define _ZIP2_
-#endif
-
-#ifdef _SIPHASH_
-/**
- * SipHash Constants
- */
-
-#define SIPHASHM_0 0x736f6d6570736575
-#define SIPHASHM_1 0x646f72616e646f6d
-#define SIPHASHM_2 0x6c7967656e657261
-#define SIPHASHM_3 0x7465646279746573
-#endif
-
-#if defined _BCRYPT_ || defined _PSAFE2_
-/**
- * bcrypt Constants
- */
-
-#define BCRYPTM_0 0x4F727068u
-#define BCRYPTM_1 0x65616E42u
-#define BCRYPTM_2 0x65686F6Cu
-#define BCRYPTM_3 0x64657253u
-#define BCRYPTM_4 0x63727944u
-#define BCRYPTM_5 0x6F756274u
-#endif
-
-#if defined _SHA1_ || defined _SAPG_ || defined _OFFICE2007_ || defined _OFFICE2010_ || defined _OLDOFFICE34_ || defined _ANDROIDFDE_ || defined _DCC2_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _PSAFE2_ || defined _LOTUS8_ || defined _PBKDF2_SHA1_ || defined _RAR3_ || defined _SHA256_SHA1_ || defined _ZIP2_
-/**
- * SHA1 Constants
- */
-
-#define SHA1M_A 0x67452301u
-#define SHA1M_B 0xefcdab89u
-#define SHA1M_C 0x98badcfeu
-#define SHA1M_D 0x10325476u
-#define SHA1M_E 0xc3d2e1f0u
-
-#define SHA1C00 0x5a827999u
-#define SHA1C01 0x6ed9eba1u
-#define SHA1C02 0x8f1bbcdcu
-#define SHA1C03 0xca62c1d6u
-#endif
-
-#if defined _SHA256_ || defined _PDF17L8_ || defined _SEVEN_ZIP_ || defined _ANDROIDFDE_ || defined _CLOUDKEY_ || defined _SCRYPT_ || defined _PBKDF2_SHA256_ || defined _SHA256_SHA1_ || defined _MS_DRSR_ || defined _ANDROIDFDE_SAMSUNG_ || defined _RAR5_ || defined _KEEPASS_
-/**
- * SHA256 Constants
- */
-
-#define SHA256M_A 0x6a09e667u
-#define SHA256M_B 0xbb67ae85u
-#define SHA256M_C 0x3c6ef372u
-#define SHA256M_D 0xa54ff53au
-#define SHA256M_E 0x510e527fu
-#define SHA256M_F 0x9b05688cu
-#define SHA256M_G 0x1f83d9abu
-#define SHA256M_H 0x5be0cd19u
-
-#define SHA256C00 0x428a2f98u
-#define SHA256C01 0x71374491u
-#define SHA256C02 0xb5c0fbcfu
-#define SHA256C03 0xe9b5dba5u
-#define SHA256C04 0x3956c25bu
-#define SHA256C05 0x59f111f1u
-#define SHA256C06 0x923f82a4u
-#define SHA256C07 0xab1c5ed5u
-#define SHA256C08 0xd807aa98u
-#define SHA256C09 0x12835b01u
-#define SHA256C0a 0x243185beu
-#define SHA256C0b 0x550c7dc3u
-#define SHA256C0c 0x72be5d74u
-#define SHA256C0d 0x80deb1feu
-#define SHA256C0e 0x9bdc06a7u
-#define SHA256C0f 0xc19bf174u
-#define SHA256C10 0xe49b69c1u
-#define SHA256C11 0xefbe4786u
-#define SHA256C12 0x0fc19dc6u
-#define SHA256C13 0x240ca1ccu
-#define SHA256C14 0x2de92c6fu
-#define SHA256C15 0x4a7484aau
-#define SHA256C16 0x5cb0a9dcu
-#define SHA256C17 0x76f988dau
-#define SHA256C18 0x983e5152u
-#define SHA256C19 0xa831c66du
-#define SHA256C1a 0xb00327c8u
-#define SHA256C1b 0xbf597fc7u
-#define SHA256C1c 0xc6e00bf3u
-#define SHA256C1d 0xd5a79147u
-#define SHA256C1e 0x06ca6351u
-#define SHA256C1f 0x14292967u
-#define SHA256C20 0x27b70a85u
-#define SHA256C21 0x2e1b2138u
-#define SHA256C22 0x4d2c6dfcu
-#define SHA256C23 0x53380d13u
-#define SHA256C24 0x650a7354u
-#define SHA256C25 0x766a0abbu
-#define SHA256C26 0x81c2c92eu
-#define SHA256C27 0x92722c85u
-#define SHA256C28 0xa2bfe8a1u
-#define SHA256C29 0xa81a664bu
-#define SHA256C2a 0xc24b8b70u
-#define SHA256C2b 0xc76c51a3u
-#define SHA256C2c 0xd192e819u
-#define SHA256C2d 0xd6990624u
-#define SHA256C2e 0xf40e3585u
-#define SHA256C2f 0x106aa070u
-#define SHA256C30 0x19a4c116u
-#define SHA256C31 0x1e376c08u
-#define SHA256C32 0x2748774cu
-#define SHA256C33 0x34b0bcb5u
-#define SHA256C34 0x391c0cb3u
-#define SHA256C35 0x4ed8aa4au
-#define SHA256C36 0x5b9cca4fu
-#define SHA256C37 0x682e6ff3u
-#define SHA256C38 0x748f82eeu
-#define SHA256C39 0x78a5636fu
-#define SHA256C3a 0x84c87814u
-#define SHA256C3b 0x8cc70208u
-#define SHA256C3c 0x90befffau
-#define SHA256C3d 0xa4506cebu
-#define SHA256C3e 0xbef9a3f7u
-#define SHA256C3f 0xc67178f2u
-#endif
-
-#if defined _MD4_ || defined _DCC2_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _MS_DRSR_ || defined _KRB5TGS_
-/**
- * MD4 Constants
- */
-
-#define MD4M_A 0x67452301u
-#define MD4M_B 0xefcdab89u
-#define MD4M_C 0x98badcfeu
-#define MD4M_D 0x10325476u
-
-#define MD4S00  3u
-#define MD4S01  7u
-#define MD4S02 11u
-#define MD4S03 19u
-#define MD4S10  3u
-#define MD4S11  5u
-#define MD4S12  9u
-#define MD4S13 13u
-#define MD4S20  3u
-#define MD4S21  9u
-#define MD4S22 11u
-#define MD4S23 15u
-
-#define MD4C00 0x00000000u
-#define MD4C01 0x5a827999u
-#define MD4C02 0x6ed9eba1u
-#endif
-
-#if defined _MD5_ || defined _MD5H_ || defined _SAPB_ || defined _OLDOFFICE01_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _NETNTLMV2_ || defined _KRB5PA_  || defined _PBKDF2_MD5_ || defined _KRB5TGS_
-/**
- * MD5 Constants
- */
-
-#define MD5M_A 0x67452301u
-#define MD5M_B 0xefcdab89u
-#define MD5M_C 0x98badcfeu
-#define MD5M_D 0x10325476u
-
-#define MD5S00  7u
-#define MD5S01 12u
-#define MD5S02 17u
-#define MD5S03 22u
-#define MD5S10  5u
-#define MD5S11  9u
-#define MD5S12 14u
-#define MD5S13 20u
-#define MD5S20  4u
-#define MD5S21 11u
-#define MD5S22 16u
-#define MD5S23 23u
-#define MD5S30  6u
-#define MD5S31 10u
-#define MD5S32 15u
-#define MD5S33 21u
-
-#define MD5C00 0xd76aa478u
-#define MD5C01 0xe8c7b756u
-#define MD5C02 0x242070dbu
-#define MD5C03 0xc1bdceeeu
-#define MD5C04 0xf57c0fafu
-#define MD5C05 0x4787c62au
-#define MD5C06 0xa8304613u
-#define MD5C07 0xfd469501u
-#define MD5C08 0x698098d8u
-#define MD5C09 0x8b44f7afu
-#define MD5C0a 0xffff5bb1u
-#define MD5C0b 0x895cd7beu
-#define MD5C0c 0x6b901122u
-#define MD5C0d 0xfd987193u
-#define MD5C0e 0xa679438eu
-#define MD5C0f 0x49b40821u
-#define MD5C10 0xf61e2562u
-#define MD5C11 0xc040b340u
-#define MD5C12 0x265e5a51u
-#define MD5C13 0xe9b6c7aau
-#define MD5C14 0xd62f105du
-#define MD5C15 0x02441453u
-#define MD5C16 0xd8a1e681u
-#define MD5C17 0xe7d3fbc8u
-#define MD5C18 0x21e1cde6u
-#define MD5C19 0xc33707d6u
-#define MD5C1a 0xf4d50d87u
-#define MD5C1b 0x455a14edu
-#define MD5C1c 0xa9e3e905u
-#define MD5C1d 0xfcefa3f8u
-#define MD5C1e 0x676f02d9u
-#define MD5C1f 0x8d2a4c8au
-#define MD5C20 0xfffa3942u
-#define MD5C21 0x8771f681u
-#define MD5C22 0x6d9d6122u
-#define MD5C23 0xfde5380cu
-#define MD5C24 0xa4beea44u
-#define MD5C25 0x4bdecfa9u
-#define MD5C26 0xf6bb4b60u
-#define MD5C27 0xbebfbc70u
-#define MD5C28 0x289b7ec6u
-#define MD5C29 0xeaa127fau
-#define MD5C2a 0xd4ef3085u
-#define MD5C2b 0x04881d05u
-#define MD5C2c 0xd9d4d039u
-#define MD5C2d 0xe6db99e5u
-#define MD5C2e 0x1fa27cf8u
-#define MD5C2f 0xc4ac5665u
-#define MD5C30 0xf4292244u
-#define MD5C31 0x432aff97u
-#define MD5C32 0xab9423a7u
-#define MD5C33 0xfc93a039u
-#define MD5C34 0x655b59c3u
-#define MD5C35 0x8f0ccc92u
-#define MD5C36 0xffeff47du
-#define MD5C37 0x85845dd1u
-#define MD5C38 0x6fa87e4fu
-#define MD5C39 0xfe2ce6e0u
-#define MD5C3a 0xa3014314u
-#define MD5C3b 0x4e0811a1u
-#define MD5C3c 0xf7537e82u
-#define MD5C3d 0xbd3af235u
-#define MD5C3e 0x2ad7d2bbu
-#define MD5C3f 0xeb86d391u
-#endif
-
-#if defined _SHA384_ || defined _PDF17L8_
-/**
- * SHA384 Constants (64 bits)
- */
-
-#define SHA384M_A 0xcbbb9d5dc1059ed8
-#define SHA384M_B 0x629a292a367cd507
-#define SHA384M_C 0x9159015a3070dd17
-#define SHA384M_D 0x152fecd8f70e5939
-#define SHA384M_E 0x67332667ffc00b31
-#define SHA384M_F 0x8eb44a8768581511
-#define SHA384M_G 0xdb0c2e0d64f98fa7
-#define SHA384M_H 0x47b5481dbefa4fa4
-
-#define SHA384C00 0x428a2f98d728ae22
-#define SHA384C01 0x7137449123ef65cd
-#define SHA384C02 0xb5c0fbcfec4d3b2f
-#define SHA384C03 0xe9b5dba58189dbbc
-#define SHA384C04 0x3956c25bf348b538
-#define SHA384C05 0x59f111f1b605d019
-#define SHA384C06 0x923f82a4af194f9b
-#define SHA384C07 0xab1c5ed5da6d8118
-#define SHA384C08 0xd807aa98a3030242
-#define SHA384C09 0x12835b0145706fbe
-#define SHA384C0a 0x243185be4ee4b28c
-#define SHA384C0b 0x550c7dc3d5ffb4e2
-#define SHA384C0c 0x72be5d74f27b896f
-#define SHA384C0d 0x80deb1fe3b1696b1
-#define SHA384C0e 0x9bdc06a725c71235
-#define SHA384C0f 0xc19bf174cf692694
-#define SHA384C10 0xe49b69c19ef14ad2
-#define SHA384C11 0xefbe4786384f25e3
-#define SHA384C12 0x0fc19dc68b8cd5b5
-#define SHA384C13 0x240ca1cc77ac9c65
-#define SHA384C14 0x2de92c6f592b0275
-#define SHA384C15 0x4a7484aa6ea6e483
-#define SHA384C16 0x5cb0a9dcbd41fbd4
-#define SHA384C17 0x76f988da831153b5
-#define SHA384C18 0x983e5152ee66dfab
-#define SHA384C19 0xa831c66d2db43210
-#define SHA384C1a 0xb00327c898fb213f
-#define SHA384C1b 0xbf597fc7beef0ee4
-#define SHA384C1c 0xc6e00bf33da88fc2
-#define SHA384C1d 0xd5a79147930aa725
-#define SHA384C1e 0x06ca6351e003826f
-#define SHA384C1f 0x142929670a0e6e70
-#define SHA384C20 0x27b70a8546d22ffc
-#define SHA384C21 0x2e1b21385c26c926
-#define SHA384C22 0x4d2c6dfc5ac42aed
-#define SHA384C23 0x53380d139d95b3df
-#define SHA384C24 0x650a73548baf63de
-#define SHA384C25 0x766a0abb3c77b2a8
-#define SHA384C26 0x81c2c92e47edaee6
-#define SHA384C27 0x92722c851482353b
-#define SHA384C28 0xa2bfe8a14cf10364
-#define SHA384C29 0xa81a664bbc423001
-#define SHA384C2a 0xc24b8b70d0f89791
-#define SHA384C2b 0xc76c51a30654be30
-#define SHA384C2c 0xd192e819d6ef5218
-#define SHA384C2d 0xd69906245565a910
-#define SHA384C2e 0xf40e35855771202a
-#define SHA384C2f 0x106aa07032bbd1b8
-#define SHA384C30 0x19a4c116b8d2d0c8
-#define SHA384C31 0x1e376c085141ab53
-#define SHA384C32 0x2748774cdf8eeb99
-#define SHA384C33 0x34b0bcb5e19b48a8
-#define SHA384C34 0x391c0cb3c5c95a63
-#define SHA384C35 0x4ed8aa4ae3418acb
-#define SHA384C36 0x5b9cca4f7763e373
-#define SHA384C37 0x682e6ff3d6b2b8a3
-#define SHA384C38 0x748f82ee5defb2fc
-#define SHA384C39 0x78a5636f43172f60
-#define SHA384C3a 0x84c87814a1f0ab72
-#define SHA384C3b 0x8cc702081a6439ec
-#define SHA384C3c 0x90befffa23631e28
-#define SHA384C3d 0xa4506cebde82bde9
-#define SHA384C3e 0xbef9a3f7b2c67915
-#define SHA384C3f 0xc67178f2e372532b
-#define SHA384C40 0xca273eceea26619c
-#define SHA384C41 0xd186b8c721c0c207
-#define SHA384C42 0xeada7dd6cde0eb1e
-#define SHA384C43 0xf57d4f7fee6ed178
-#define SHA384C44 0x06f067aa72176fba
-#define SHA384C45 0x0a637dc5a2c898a6
-#define SHA384C46 0x113f9804bef90dae
-#define SHA384C47 0x1b710b35131c471b
-#define SHA384C48 0x28db77f523047d84
-#define SHA384C49 0x32caab7b40c72493
-#define SHA384C4a 0x3c9ebe0a15c9bebc
-#define SHA384C4b 0x431d67c49c100d4c
-#define SHA384C4c 0x4cc5d4becb3e42b6
-#define SHA384C4d 0x597f299cfc657e2a
-#define SHA384C4e 0x5fcb6fab3ad6faec
-#define SHA384C4f 0x6c44198c4a475817
-
-#endif
-
-#if defined _SHA512_ || defined _CLOUDKEY_ || defined _OFFICE2013_ || defined _PDF17L8_ || defined _PBKDF2_SHA512_
-/**
- * SHA512 Constants (64 bits)
- */
-
-#define SHA512M_A 0x6a09e667f3bcc908
-#define SHA512M_B 0xbb67ae8584caa73b
-#define SHA512M_C 0x3c6ef372fe94f82b
-#define SHA512M_D 0xa54ff53a5f1d36f1
-#define SHA512M_E 0x510e527fade682d1
-#define SHA512M_F 0x9b05688c2b3e6c1f
-#define SHA512M_G 0x1f83d9abfb41bd6b
-#define SHA512M_H 0x5be0cd19137e2179
-
-#define SHA512C00 0x428a2f98d728ae22
-#define SHA512C01 0x7137449123ef65cd
-#define SHA512C02 0xb5c0fbcfec4d3b2f
-#define SHA512C03 0xe9b5dba58189dbbc
-#define SHA512C04 0x3956c25bf348b538
-#define SHA512C05 0x59f111f1b605d019
-#define SHA512C06 0x923f82a4af194f9b
-#define SHA512C07 0xab1c5ed5da6d8118
-#define SHA512C08 0xd807aa98a3030242
-#define SHA512C09 0x12835b0145706fbe
-#define SHA512C0a 0x243185be4ee4b28c
-#define SHA512C0b 0x550c7dc3d5ffb4e2
-#define SHA512C0c 0x72be5d74f27b896f
-#define SHA512C0d 0x80deb1fe3b1696b1
-#define SHA512C0e 0x9bdc06a725c71235
-#define SHA512C0f 0xc19bf174cf692694
-#define SHA512C10 0xe49b69c19ef14ad2
-#define SHA512C11 0xefbe4786384f25e3
-#define SHA512C12 0x0fc19dc68b8cd5b5
-#define SHA512C13 0x240ca1cc77ac9c65
-#define SHA512C14 0x2de92c6f592b0275
-#define SHA512C15 0x4a7484aa6ea6e483
-#define SHA512C16 0x5cb0a9dcbd41fbd4
-#define SHA512C17 0x76f988da831153b5
-#define SHA512C18 0x983e5152ee66dfab
-#define SHA512C19 0xa831c66d2db43210
-#define SHA512C1a 0xb00327c898fb213f
-#define SHA512C1b 0xbf597fc7beef0ee4
-#define SHA512C1c 0xc6e00bf33da88fc2
-#define SHA512C1d 0xd5a79147930aa725
-#define SHA512C1e 0x06ca6351e003826f
-#define SHA512C1f 0x142929670a0e6e70
-#define SHA512C20 0x27b70a8546d22ffc
-#define SHA512C21 0x2e1b21385c26c926
-#define SHA512C22 0x4d2c6dfc5ac42aed
-#define SHA512C23 0x53380d139d95b3df
-#define SHA512C24 0x650a73548baf63de
-#define SHA512C25 0x766a0abb3c77b2a8
-#define SHA512C26 0x81c2c92e47edaee6
-#define SHA512C27 0x92722c851482353b
-#define SHA512C28 0xa2bfe8a14cf10364
-#define SHA512C29 0xa81a664bbc423001
-#define SHA512C2a 0xc24b8b70d0f89791
-#define SHA512C2b 0xc76c51a30654be30
-#define SHA512C2c 0xd192e819d6ef5218
-#define SHA512C2d 0xd69906245565a910
-#define SHA512C2e 0xf40e35855771202a
-#define SHA512C2f 0x106aa07032bbd1b8
-#define SHA512C30 0x19a4c116b8d2d0c8
-#define SHA512C31 0x1e376c085141ab53
-#define SHA512C32 0x2748774cdf8eeb99
-#define SHA512C33 0x34b0bcb5e19b48a8
-#define SHA512C34 0x391c0cb3c5c95a63
-#define SHA512C35 0x4ed8aa4ae3418acb
-#define SHA512C36 0x5b9cca4f7763e373
-#define SHA512C37 0x682e6ff3d6b2b8a3
-#define SHA512C38 0x748f82ee5defb2fc
-#define SHA512C39 0x78a5636f43172f60
-#define SHA512C3a 0x84c87814a1f0ab72
-#define SHA512C3b 0x8cc702081a6439ec
-#define SHA512C3c 0x90befffa23631e28
-#define SHA512C3d 0xa4506cebde82bde9
-#define SHA512C3e 0xbef9a3f7b2c67915
-#define SHA512C3f 0xc67178f2e372532b
-#define SHA512C40 0xca273eceea26619c
-#define SHA512C41 0xd186b8c721c0c207
-#define SHA512C42 0xeada7dd6cde0eb1e
-#define SHA512C43 0xf57d4f7fee6ed178
-#define SHA512C44 0x06f067aa72176fba
-#define SHA512C45 0x0a637dc5a2c898a6
-#define SHA512C46 0x113f9804bef90dae
-#define SHA512C47 0x1b710b35131c471b
-#define SHA512C48 0x28db77f523047d84
-#define SHA512C49 0x32caab7b40c72493
-#define SHA512C4a 0x3c9ebe0a15c9bebc
-#define SHA512C4b 0x431d67c49c100d4c
-#define SHA512C4c 0x4cc5d4becb3e42b6
-#define SHA512C4d 0x597f299cfc657e2a
-#define SHA512C4e 0x5fcb6fab3ad6faec
-#define SHA512C4f 0x6c44198c4a475817
-
-#define SHA512REV0 0x5218a97a1b97e8a0
-#define SHA512REV1 0x4334c1bea164f555
-
-#endif
-
-#ifdef _RIPEMD160_
-/**
- * RIPEMD160 Constants
- */
-
-#define RIPEMD160M_A 0x67452301u
-#define RIPEMD160M_B 0xefcdab89u
-#define RIPEMD160M_C 0x98badcfeu
-#define RIPEMD160M_D 0x10325476u
-#define RIPEMD160M_E 0xc3d2e1f0u
-
-#define RIPEMD160C00 0x00000000u
-#define RIPEMD160C10 0x5a827999u
-#define RIPEMD160C20 0x6ed9eba1u
-#define RIPEMD160C30 0x8f1bbcdcu
-#define RIPEMD160C40 0xa953fd4eu
-#define RIPEMD160C50 0x50a28be6u
-#define RIPEMD160C60 0x5c4dd124u
-#define RIPEMD160C70 0x6d703ef3u
-#define RIPEMD160C80 0x7a6d76e9u
-#define RIPEMD160C90 0x00000000u
-
-#define RIPEMD160S00 11u
-#define RIPEMD160S01 14u
-#define RIPEMD160S02 15u
-#define RIPEMD160S03 12u
-#define RIPEMD160S04  5u
-#define RIPEMD160S05  8u
-#define RIPEMD160S06  7u
-#define RIPEMD160S07  9u
-#define RIPEMD160S08 11u
-#define RIPEMD160S09 13u
-#define RIPEMD160S0A 14u
-#define RIPEMD160S0B 15u
-#define RIPEMD160S0C  6u
-#define RIPEMD160S0D  7u
-#define RIPEMD160S0E  9u
-#define RIPEMD160S0F  8u
-
-#define RIPEMD160S10  7u
-#define RIPEMD160S11  6u
-#define RIPEMD160S12  8u
-#define RIPEMD160S13 13u
-#define RIPEMD160S14 11u
-#define RIPEMD160S15  9u
-#define RIPEMD160S16  7u
-#define RIPEMD160S17 15u
-#define RIPEMD160S18  7u
-#define RIPEMD160S19 12u
-#define RIPEMD160S1A 15u
-#define RIPEMD160S1B  9u
-#define RIPEMD160S1C 11u
-#define RIPEMD160S1D  7u
-#define RIPEMD160S1E 13u
-#define RIPEMD160S1F 12u
-
-#define RIPEMD160S20 11u
-#define RIPEMD160S21 13u
-#define RIPEMD160S22  6u
-#define RIPEMD160S23  7u
-#define RIPEMD160S24 14u
-#define RIPEMD160S25  9u
-#define RIPEMD160S26 13u
-#define RIPEMD160S27 15u
-#define RIPEMD160S28 14u
-#define RIPEMD160S29  8u
-#define RIPEMD160S2A 13u
-#define RIPEMD160S2B  6u
-#define RIPEMD160S2C  5u
-#define RIPEMD160S2D 12u
-#define RIPEMD160S2E  7u
-#define RIPEMD160S2F  5u
-
-#define RIPEMD160S30 11u
-#define RIPEMD160S31 12u
-#define RIPEMD160S32 14u
-#define RIPEMD160S33 15u
-#define RIPEMD160S34 14u
-#define RIPEMD160S35 15u
-#define RIPEMD160S36  9u
-#define RIPEMD160S37  8u
-#define RIPEMD160S38  9u
-#define RIPEMD160S39 14u
-#define RIPEMD160S3A  5u
-#define RIPEMD160S3B  6u
-#define RIPEMD160S3C  8u
-#define RIPEMD160S3D  6u
-#define RIPEMD160S3E  5u
-#define RIPEMD160S3F 12u
-
-#define RIPEMD160S40  9u
-#define RIPEMD160S41 15u
-#define RIPEMD160S42  5u
-#define RIPEMD160S43 11u
-#define RIPEMD160S44  6u
-#define RIPEMD160S45  8u
-#define RIPEMD160S46 13u
-#define RIPEMD160S47 12u
-#define RIPEMD160S48  5u
-#define RIPEMD160S49 12u
-#define RIPEMD160S4A 13u
-#define RIPEMD160S4B 14u
-#define RIPEMD160S4C 11u
-#define RIPEMD160S4D  8u
-#define RIPEMD160S4E  5u
-#define RIPEMD160S4F  6u
-
-#define RIPEMD160S50  8u
-#define RIPEMD160S51  9u
-#define RIPEMD160S52  9u
-#define RIPEMD160S53 11u
-#define RIPEMD160S54 13u
-#define RIPEMD160S55 15u
-#define RIPEMD160S56 15u
-#define RIPEMD160S57  5u
-#define RIPEMD160S58  7u
-#define RIPEMD160S59  7u
-#define RIPEMD160S5A  8u
-#define RIPEMD160S5B 11u
-#define RIPEMD160S5C 14u
-#define RIPEMD160S5D 14u
-#define RIPEMD160S5E 12u
-#define RIPEMD160S5F  6u
-
-#define RIPEMD160S60  9u
-#define RIPEMD160S61 13u
-#define RIPEMD160S62 15u
-#define RIPEMD160S63  7u
-#define RIPEMD160S64 12u
-#define RIPEMD160S65  8u
-#define RIPEMD160S66  9u
-#define RIPEMD160S67 11u
-#define RIPEMD160S68  7u
-#define RIPEMD160S69  7u
-#define RIPEMD160S6A 12u
-#define RIPEMD160S6B  7u
-#define RIPEMD160S6C  6u
-#define RIPEMD160S6D 15u
-#define RIPEMD160S6E 13u
-#define RIPEMD160S6F 11u
-
-#define RIPEMD160S70  9u
-#define RIPEMD160S71  7u
-#define RIPEMD160S72 15u
-#define RIPEMD160S73 11u
-#define RIPEMD160S74  8u
-#define RIPEMD160S75  6u
-#define RIPEMD160S76  6u
-#define RIPEMD160S77 14u
-#define RIPEMD160S78 12u
-#define RIPEMD160S79 13u
-#define RIPEMD160S7A  5u
-#define RIPEMD160S7B 14u
-#define RIPEMD160S7C 13u
-#define RIPEMD160S7D 13u
-#define RIPEMD160S7E  7u
-#define RIPEMD160S7F  5u
-
-#define RIPEMD160S80 15u
-#define RIPEMD160S81  5u
-#define RIPEMD160S82  8u
-#define RIPEMD160S83 11u
-#define RIPEMD160S84 14u
-#define RIPEMD160S85 14u
-#define RIPEMD160S86  6u
-#define RIPEMD160S87 14u
-#define RIPEMD160S88  6u
-#define RIPEMD160S89  9u
-#define RIPEMD160S8A 12u
-#define RIPEMD160S8B  9u
-#define RIPEMD160S8C 12u
-#define RIPEMD160S8D  5u
-#define RIPEMD160S8E 15u
-#define RIPEMD160S8F  8u
-
-#define RIPEMD160S90  8u
-#define RIPEMD160S91  5u
-#define RIPEMD160S92 12u
-#define RIPEMD160S93  9u
-#define RIPEMD160S94 12u
-#define RIPEMD160S95  5u
-#define RIPEMD160S96 14u
-#define RIPEMD160S97  6u
-#define RIPEMD160S98  8u
-#define RIPEMD160S99 13u
-#define RIPEMD160S9A  6u
-#define RIPEMD160S9B  5u
-#define RIPEMD160S9C 15u
-#define RIPEMD160S9D 13u
-#define RIPEMD160S9E 11u
-#define RIPEMD160S9F 11u
-
-#endif
-
-#ifdef _KECCAK_
-/**
- * KECCAK Constants
- */
-
-#define KECCAK_RNDC_00 0x0000000000000001
-#define KECCAK_RNDC_01 0x0000000000008082
-#define KECCAK_RNDC_02 0x000000000000808a
-#define KECCAK_RNDC_03 0x0000000080008000
-#define KECCAK_RNDC_04 0x000000000000808b
-#define KECCAK_RNDC_05 0x0000000080000001
-#define KECCAK_RNDC_06 0x0000000080008081
-#define KECCAK_RNDC_07 0x0000000000008009
-#define KECCAK_RNDC_08 0x000000000000008a
-#define KECCAK_RNDC_09 0x0000000000000088
-#define KECCAK_RNDC_10 0x0000000080008009
-#define KECCAK_RNDC_11 0x000000008000000a
-#define KECCAK_RNDC_12 0x000000008000808b
-#define KECCAK_RNDC_13 0x000000000000008b
-#define KECCAK_RNDC_14 0x0000000000008089
-#define KECCAK_RNDC_15 0x0000000000008003
-#define KECCAK_RNDC_16 0x0000000000008002
-#define KECCAK_RNDC_17 0x0000000000000080
-#define KECCAK_RNDC_18 0x000000000000800a
-#define KECCAK_RNDC_19 0x000000008000000a
-#define KECCAK_RNDC_20 0x0000000080008081
-#define KECCAK_RNDC_21 0x0000000000008080
-#define KECCAK_RNDC_22 0x0000000080000001
-#define KECCAK_RNDC_23 0x0000000080008008
-
-#define KECCAK_PILN_00 10
-#define KECCAK_PILN_01  7
-#define KECCAK_PILN_02 11
-#define KECCAK_PILN_03 17
-#define KECCAK_PILN_04 18
-#define KECCAK_PILN_05  3
-#define KECCAK_PILN_06  5
-#define KECCAK_PILN_07 16
-#define KECCAK_PILN_08  8
-#define KECCAK_PILN_09 21
-#define KECCAK_PILN_10 24
-#define KECCAK_PILN_11  4
-#define KECCAK_PILN_12 15
-#define KECCAK_PILN_13 23
-#define KECCAK_PILN_14 19
-#define KECCAK_PILN_15 13
-#define KECCAK_PILN_16 12
-#define KECCAK_PILN_17  2
-#define KECCAK_PILN_18 20
-#define KECCAK_PILN_19 14
-#define KECCAK_PILN_20 22
-#define KECCAK_PILN_21  9
-#define KECCAK_PILN_22  6
-#define KECCAK_PILN_23  1
-
-#define KECCAK_ROTC_00  1
-#define KECCAK_ROTC_01  3
-#define KECCAK_ROTC_02  6
-#define KECCAK_ROTC_03 10
-#define KECCAK_ROTC_04 15
-#define KECCAK_ROTC_05 21
-#define KECCAK_ROTC_06 28
-#define KECCAK_ROTC_07 36
-#define KECCAK_ROTC_08 45
-#define KECCAK_ROTC_09 55
-#define KECCAK_ROTC_10  2
-#define KECCAK_ROTC_11 14
-#define KECCAK_ROTC_12 27
-#define KECCAK_ROTC_13 41
-#define KECCAK_ROTC_14 56
-#define KECCAK_ROTC_15  8
-#define KECCAK_ROTC_16 25
-#define KECCAK_ROTC_17 43
-#define KECCAK_ROTC_18 62
-#define KECCAK_ROTC_19 18
-#define KECCAK_ROTC_20 39
-#define KECCAK_ROTC_21 61
-#define KECCAK_ROTC_22 20
-#define KECCAK_ROTC_23 44
-
-
-#endif
-
-#ifdef _MYSQL323_
-/**
- * MYSQL323 Constants
- */
-
-#define MYSQL323_A 0x50305735
-#define MYSQL323_B 0x12345671
-
-#endif

diff --git a/include/kernel_functions.c b/include/kernel_functions.c
deleted file mode 100644 (file)
index 0ccbb25..0000000
--- a/include/kernel_functions.c
+++ /dev/null
@@ -1,413 +0,0 @@
-/**
- * Author......: Jens Steube <jens.steube@gmail.com>
- * License.....: MIT
- */
-
-#if defined _MD4_ || defined _DCC2_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _MS_DRSR_ || defined _KRB5TGS_
-
-#define MD4_F_S(x,y,z)  (((x) & (y)) | ((~(x)) & (z)))
-#define MD4_G_S(x,y,z)  (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
-#define MD4_H_S(x,y,z)  ((x) ^ (y) ^ (z))
-
-#ifdef IS_NV
-#define MD4_F(x,y,z)    (((x) & (y)) | ((~(x)) & (z)))
-#define MD4_G(x,y,z)    (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
-#define MD4_H(x,y,z)    ((x) ^ (y) ^ (z))
-#define MD4_Fo(x,y,z)   (MD4_F((x), (y), (z)))
-#define MD4_Go(x,y,z)   (MD4_G((x), (y), (z)))
-#endif
-
-#ifdef IS_AMD
-#define MD4_F(x,y,z)    (((x) & (y)) | ((~(x)) & (z)))
-#define MD4_G(x,y,z)    (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
-#define MD4_H(x,y,z)    ((x) ^ (y) ^ (z))
-#define MD4_Fo(x,y,z)   (bitselect ((z), (y), (x)))
-#define MD4_Go(x,y,z)   (bitselect ((x), (y), ((x) ^ (z))))
-#endif
-
-#ifdef IS_GENERIC
-#define MD4_F(x,y,z)    (((x) & (y)) | ((~(x)) & (z)))
-#define MD4_G(x,y,z)    (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
-#define MD4_H(x,y,z)    ((x) ^ (y) ^ (z))
-#define MD4_Fo(x,y,z)   (MD4_F((x), (y), (z)))
-#define MD4_Go(x,y,z)   (MD4_G((x), (y), (z)))
-#endif
-
-#define MD4_STEP_S(f,a,b,c,d,x,K,s) \
-{                                   \
-  a += K;                           \
-  a += x;                           \
-  a += f (b, c, d);                 \
-  a  = rotl32_S (a, s);             \
-}
-
-#define MD4_STEP(f,a,b,c,d,x,K,s)   \
-{                                   \
-  a += K;                           \
-  a += x;                           \
-  a += f (b, c, d);                 \
-  a  = rotl32 (a, s);               \
-}
-
-#define MD4_STEP0(f,a,b,c,d,K,s)    \
-{                                   \
-  a += K;                           \
-  a += f (b, c, d);                 \
-  a  = rotl32 (a, s);               \
-}
-
-#endif
-
-#if defined _MD5_ || defined _MD5H_ || defined _SAPB_ || defined _OLDOFFICE01_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _NETNTLMV2_ || defined _KRB5PA_ || defined _PBKDF2_MD5_ || defined _KRB5TGS_
-
-#define MD5_F_S(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
-#define MD5_G_S(x,y,z)  ((y) ^ ((z) & ((x) ^ (y))))
-#define MD5_H_S(x,y,z)  ((x) ^ (y) ^ (z))
-#define MD5_I_S(x,y,z)  ((y) ^ ((x) | ~(z)))
-
-#ifdef IS_NV
-#define MD5_F(x,y,z)    ((z) ^ ((x) & ((y) ^ (z))))
-#define MD5_G(x,y,z)    ((y) ^ ((z) & ((x) ^ (y))))
-#define MD5_H(x,y,z)    ((x) ^ (y) ^ (z))
-#define MD5_I(x,y,z)    ((y) ^ ((x) | ~(z)))
-#define MD5_Fo(x,y,z)   (MD5_F((x), (y), (z)))
-#define MD5_Go(x,y,z)   (MD5_G((x), (y), (z)))
-#endif
-
-#ifdef IS_AMD
-#define MD5_F(x,y,z)    ((z) ^ ((x) & ((y) ^ (z))))
-#define MD5_G(x,y,z)    ((y) ^ ((z) & ((x) ^ (y))))
-#define MD5_H(x,y,z)    ((x) ^ (y) ^ (z))
-#define MD5_I(x,y,z)    (bitselect (0xffffffffU, (x), (z)) ^ (y))
-#define MD5_Fo(x,y,z)   (bitselect ((z), (y), (x)))
-#define MD5_Go(x,y,z)   (bitselect ((y), (x), (z)))
-#endif
-
-#ifdef IS_GENERIC
-#define MD5_F(x,y,z)    ((z) ^ ((x) & ((y) ^ (z))))
-#define MD5_G(x,y,z)    ((y) ^ ((z) & ((x) ^ (y))))
-#define MD5_H(x,y,z)    ((x) ^ (y) ^ (z))
-#define MD5_I(x,y,z)    ((y) ^ ((x) | ~(z)))
-#define MD5_Fo(x,y,z)   (MD5_F((x), (y), (z)))
-#define MD5_Go(x,y,z)   (MD5_G((x), (y), (z)))
-#endif
-
-#define MD5_STEP_S(f,a,b,c,d,x,K,s) \
-{                                   \
-  a += K;                           \
-  a += x;                           \
-  a += f (b, c, d);                 \
-  a  = rotl32_S (a, s);             \
-  a += b;                           \
-}
-
-#define MD5_STEP(f,a,b,c,d,x,K,s)   \
-{                                   \
-  a += K;                           \
-  a += x;                           \
-  a += f (b, c, d);                 \
-  a  = rotl32 (a, s);               \
-  a += b;                           \
-}
-
-#define MD5_STEP0(f,a,b,c,d,K,s)    \
-{                                   \
-  a += K;                           \
-  a += f (b, c, d);                 \
-  a  = rotl32 (a, s);               \
-  a += b;                           \
-}
-#endif
-
-#if defined _SHA1_ || defined _SAPG_ || defined _OFFICE2007_ || defined _OFFICE2010_ || defined _OLDOFFICE34_ || defined _ANDROIDFDE_ || defined _DCC2_ || defined _WPA_ || defined _MD5_SHA1_ || defined _SHA1_MD5_ || defined _PSAFE2_ || defined _LOTUS8_ || defined _PBKDF2_SHA1_ || defined _RAR3_ || defined _SHA256_SHA1_ || defined _ZIP2_
-
-#ifdef IS_NV
-#define SHA1_F0(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
-#define SHA1_F1(x,y,z)  ((x) ^ (y) ^ (z))
-#define SHA1_F2(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
-#define SHA1_F0o(x,y,z) (SHA1_F0 ((x), (y), (z)))
-#define SHA1_F2o(x,y,z) (SHA1_F2 ((x), (y), (z)))
-#endif
-
-#ifdef IS_AMD
-#define SHA1_F0(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
-#define SHA1_F1(x,y,z)  ((x) ^ (y) ^ (z))
-#define SHA1_F2(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
-#define SHA1_F0o(x,y,z) (bitselect ((z), (y), (x)))
-#define SHA1_F2o(x,y,z) (bitselect ((x), (y), ((x) ^ (z))))
-#endif
-
-#ifdef IS_GENERIC
-#define SHA1_F0(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
-#define SHA1_F1(x,y,z)  ((x) ^ (y) ^ (z))
-#define SHA1_F2(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
-#define SHA1_F0o(x,y,z) (SHA1_F0 ((x), (y), (z)))
-#define SHA1_F2o(x,y,z) (SHA1_F2 ((x), (y), (z)))
-#endif
-
-#define SHA1_STEP_S(f,a,b,c,d,e,x)  \
-{                                   \
-  e += K;                           \
-  e += x;                           \
-  e += f (b, c, d);                 \
-  e += rotl32_S (a,  5u);           \
-  b  = rotl32_S (b, 30u);           \
-}
-
-#define SHA1_STEP(f,a,b,c,d,e,x)    \
-{                                   \
-  e += K;                           \
-  e += x;                           \
-  e += f (b, c, d);                 \
-  e += rotl32 (a,  5u);             \
-  b  = rotl32 (b, 30u);             \
-}
-
-#define SHA1_STEP0(f,a,b,c,d,e,x)   \
-{                                   \
-  e += K;                           \
-  e += f (b, c, d);                 \
-  e += rotl32 (a,  5u);             \
-  b  = rotl32 (b, 30u);             \
-}
-
-#define SHA1_STEPX(f,a,b,c,d,e,x)   \
-{                                   \
-  e += x;                           \
-  e += f (b, c, d);                 \
-  e += rotl32 (a,  5u);             \
-  b  = rotl32 (b, 30u);             \
-}
-
-#define SHA1_STEP_PE(f,a,b,c,d,e,x) \
-{                                   \
-  e += x;                           \
-  e += f (b, c, d);                 \
-  e += rotl32 (a,  5u);             \
-}
-
-#define SHA1_STEP_PB(f,a,b,c,d,e,x) \
-{                                   \
-  e += K;                           \
-  b  = rotl32 (b, 30u);             \
-}
-#endif
-
-#if defined _SHA256_ || defined _PDF17L8_ || defined _SEVEN_ZIP_ || defined _ANDROIDFDE_ || defined _CLOUDKEY_ || defined _SCRYPT_ || defined _PBKDF2_SHA256_  || defined _SHA256_SHA1_ || defined _MS_DRSR_ || defined _ANDROIDFDE_SAMSUNG_ || defined _RAR5_ || defined _KEEPASS_
-
-#define SHIFT_RIGHT_32(x,n) ((x) >> (n))
-
-#define SHA256_S0_S(x) (rotl32_S ((x), 25u) ^ rotl32_S ((x), 14u) ^ SHIFT_RIGHT_32 ((x),  3u))
-#define SHA256_S1_S(x) (rotl32_S ((x), 15u) ^ rotl32_S ((x), 13u) ^ SHIFT_RIGHT_32 ((x), 10u))
-#define SHA256_S2_S(x) (rotl32_S ((x), 30u) ^ rotl32_S ((x), 19u) ^ rotl32_S ((x), 10u))
-#define SHA256_S3_S(x) (rotl32_S ((x), 26u) ^ rotl32_S ((x), 21u) ^ rotl32_S ((x),  7u))
-
-#define SHA256_S0(x) (rotl32 ((x), 25u) ^ rotl32 ((x), 14u) ^ SHIFT_RIGHT_32 ((x),  3u))
-#define SHA256_S1(x) (rotl32 ((x), 15u) ^ rotl32 ((x), 13u) ^ SHIFT_RIGHT_32 ((x), 10u))
-#define SHA256_S2(x) (rotl32 ((x), 30u) ^ rotl32 ((x), 19u) ^ rotl32 ((x), 10u))
-#define SHA256_S3(x) (rotl32 ((x), 26u) ^ rotl32 ((x), 21u) ^ rotl32 ((x),  7u))
-
-#ifdef IS_NV
-#define SHA256_F0(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
-#define SHA256_F1(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
-#define SHA256_F0o(x,y,z) (SHA256_F0 ((x), (y), (z)))
-#define SHA256_F1o(x,y,z) (SHA256_F1 ((x), (y), (z)))
-#endif
-
-#ifdef IS_AMD
-#define SHA256_F0(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
-#define SHA256_F1(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
-#define SHA256_F0o(x,y,z) (bitselect ((x), (y), ((x) ^ (z))))
-#define SHA256_F1o(x,y,z) (bitselect ((z), (y), (x)))
-#endif
-
-#ifdef IS_GENERIC
-#define SHA256_F0(x,y,z)  (((x) & (y)) | ((z) & ((x) ^ (y))))
-#define SHA256_F1(x,y,z)  ((z) ^ ((x) & ((y) ^ (z))))
-#define SHA256_F0o(x,y,z) (SHA256_F0 ((x), (y), (z)))
-#define SHA256_F1o(x,y,z) (SHA256_F1 ((x), (y), (z)))
-#endif
-
-#define SHA256_STEP_S(F0,F1,a,b,c,d,e,f,g,h,x,K)  \
-{                                                 \
-  h += K;                                         \
-  h += x;                                         \
-  h += SHA256_S3_S (e);                           \
-  h += F1 (e,f,g);                                \
-  d += h;                                         \
-  h += SHA256_S2_S (a);                           \
-  h += F0 (a,b,c);                                \
-}
-
-#define SHA256_EXPAND_S(x,y,z,w) (SHA256_S1_S (x) + y + SHA256_S0_S (z) + w)
-
-#define SHA256_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K)    \
-{                                                 \
-  h += K;                                         \
-  h += x;                                         \
-  h += SHA256_S3 (e);                             \
-  h += F1 (e,f,g);                                \
-  d += h;                                         \
-  h += SHA256_S2 (a);                             \
-  h += F0 (a,b,c);                                \
-}
-
-#define SHA256_EXPAND(x,y,z,w) (SHA256_S1 (x) + y + SHA256_S0 (z) + w)
-
-#endif
-
-#if defined _SHA384_ || defined _PDF17L8_
-
-#define SHIFT_RIGHT_64(x,n) ((x) >> (n))
-
-#define SHA384_S0(x) (rotr64 ((x), 28) ^ rotr64 ((x), 34) ^ rotr64 ((x), 39))
-#define SHA384_S1(x) (rotr64 ((x), 14) ^ rotr64 ((x), 18) ^ rotr64 ((x), 41))
-#define SHA384_S2(x) (rotr64 ((x),  1) ^ rotr64 ((x),  8) ^ SHIFT_RIGHT_64 ((x), 7))
-#define SHA384_S3(x) (rotr64 ((x), 19) ^ rotr64 ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6))
-
-#define SHA384_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z))))
-#define SHA384_F1(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y))))
-
-#ifdef IS_NV
-#define SHA384_F0o(x,y,z) (SHA384_F0 ((x), (y), (z)))
-#define SHA384_F1o(x,y,z) (SHA384_F1 ((x), (y), (z)))
-#endif
-
-#ifdef IS_AMD
-#define SHA384_F0o(x,y,z) (bitselect ((z), (y), (x)))
-#define SHA384_F1o(x,y,z) (bitselect ((x), (y), ((x) ^ (z))))
-#endif
-
-#ifdef IS_GENERIC
-#define SHA384_F0o(x,y,z) (SHA384_F0 ((x), (y), (z)))
-#define SHA384_F1o(x,y,z) (SHA384_F1 ((x), (y), (z)))
-#endif
-
-#define SHA384_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K)  \
-{                                               \
-  h += K;                                       \
-  h += x;                                       \
-  h += SHA384_S1 (e);                           \
-  h += F0 (e, f, g);                            \
-  d += h;                                       \
-  h += SHA384_S0 (a);                           \
-  h += F1 (a, b, c);                            \
-}
-
-#define SHA384_EXPAND(x,y,z,w) (SHA384_S3 (x) + y + SHA384_S2 (z) + w)
-#endif
-
-#if defined _SHA512_ || defined _CLOUDKEY_ || defined _OFFICE2013_ || defined _PDF17L8_ || defined _PBKDF2_SHA512_
-
-#define SHIFT_RIGHT_64(x,n) ((x) >> (n))
-
-#define SHA512_S0_S(x) (rotr64_S ((x), 28) ^ rotr64_S ((x), 34) ^ rotr64_S ((x), 39))
-#define SHA512_S1_S(x) (rotr64_S ((x), 14) ^ rotr64_S ((x), 18) ^ rotr64_S ((x), 41))
-#define SHA512_S2_S(x) (rotr64_S ((x),  1) ^ rotr64_S ((x),  8) ^ SHIFT_RIGHT_64 ((x), 7))
-#define SHA512_S3_S(x) (rotr64_S ((x), 19) ^ rotr64_S ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6))
-
-#define SHA512_S0(x) (rotr64 ((x), 28) ^ rotr64 ((x), 34) ^ rotr64 ((x), 39))
-#define SHA512_S1(x) (rotr64 ((x), 14) ^ rotr64 ((x), 18) ^ rotr64 ((x), 41))
-#define SHA512_S2(x) (rotr64 ((x),  1) ^ rotr64 ((x),  8) ^ SHIFT_RIGHT_64 ((x), 7))
-#define SHA512_S3(x) (rotr64 ((x), 19) ^ rotr64 ((x), 61) ^ SHIFT_RIGHT_64 ((x), 6))
-
-#define SHA512_F0(x,y,z) ((z) ^ ((x) & ((y) ^ (z))))
-#define SHA512_F1(x,y,z) (((x) & (y)) | ((z) & ((x) ^ (y))))
-
-#ifdef IS_NV
-#define SHA512_F0o(x,y,z) (SHA512_F0 ((x), (y), (z)))
-#define SHA512_F1o(x,y,z) (SHA512_F1 ((x), (y), (z)))
-#endif
-
-#ifdef IS_AMD
-#define SHA512_F0o(x,y,z) (bitselect ((z), (y), (x)))
-#define SHA512_F1o(x,y,z) (bitselect ((x), (y), ((x) ^ (z))))
-#endif
-
-#ifdef IS_GENERIC
-#define SHA512_F0o(x,y,z) (SHA512_F0 ((x), (y), (z)))
-#define SHA512_F1o(x,y,z) (SHA512_F1 ((x), (y), (z)))
-#endif
-
-#define SHA512_STEP_S(F0,F1,a,b,c,d,e,f,g,h,x,K)  \
-{                                                 \
-  h += K;                                         \
-  h += x;                                         \
-  h += SHA512_S1_S (e);                           \
-  h += F0 (e, f, g);                              \
-  d += h;                                         \
-  h += SHA512_S0_S (a);                           \
-  h += F1 (a, b, c);                              \
-}
-
-#define SHA512_EXPAND_S(x,y,z,w) (SHA512_S3_S (x) + y + SHA512_S2_S (z) + w)
-
-#define SHA512_STEP(F0,F1,a,b,c,d,e,f,g,h,x,K)  \
-{                                               \
-  h += K;                                       \
-  h += x;                                       \
-  h += SHA512_S1 (e);                           \
-  h += F0 (e, f, g);                            \
-  d += h;                                       \
-  h += SHA512_S0 (a);                           \
-  h += F1 (a, b, c);                            \
-}
-
-#define SHA512_EXPAND(x,y,z,w) (SHA512_S3 (x) + y + SHA512_S2 (z) + w)
-#endif
-
-#ifdef _RIPEMD160_
-
-#ifdef IS_NV
-#define RIPEMD160_F(x,y,z)    ((x) ^ (y) ^ (z))
-#define RIPEMD160_G(x,y,z)    ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */
-#define RIPEMD160_H(x,y,z)    (((x) | ~(y)) ^ (z))
-#define RIPEMD160_I(x,y,z)    ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */
-#define RIPEMD160_J(x,y,z)    ((x) ^ ((y) | ~(z)))
-#define RIPEMD160_Go(x,y,z)   (RIPEMD160_G ((x), (y), (z)))
-#define RIPEMD160_Io(x,y,z)   (RIPEMD160_I ((x), (y), (z)))
-#endif
-
-#ifdef IS_AMD
-#define RIPEMD160_F(x,y,z)    ((x) ^ (y) ^ (z))
-#define RIPEMD160_G(x,y,z)    ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */
-#define RIPEMD160_H(x,y,z)    (((x) | ~(y)) ^ (z))
-#define RIPEMD160_I(x,y,z)    ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */
-#define RIPEMD160_J(x,y,z)    ((x) ^ ((y) | ~(z)))
-#define RIPEMD160_Go(x,y,z)   (bitselect ((z), (y), (x)))
-#define RIPEMD160_Io(x,y,z)   (bitselect ((y), (x), (z)))
-#endif
-
-#ifdef IS_GENERIC
-#define RIPEMD160_F(x,y,z)    ((x) ^ (y) ^ (z))
-#define RIPEMD160_G(x,y,z)    ((z) ^ ((x) & ((y) ^ (z)))) /* x ? y : z */
-#define RIPEMD160_H(x,y,z)    (((x) | ~(y)) ^ (z))
-#define RIPEMD160_I(x,y,z)    ((y) ^ ((z) & ((x) ^ (y)))) /* z ? x : y */
-#define RIPEMD160_J(x,y,z)    ((x) ^ ((y) | ~(z)))
-#define RIPEMD160_Go(x,y,z)   (RIPEMD160_G ((x), (y), (z)))
-#define RIPEMD160_Io(x,y,z)   (RIPEMD160_I ((x), (y), (z)))
-#endif
-
-#define RIPEMD160_STEP(f,a,b,c,d,e,x,K,s) \
-{                                         \
-  a += K;                                 \
-  a += x;                                 \
-  a += f (b, c, d);                       \
-  a  = rotl32 (a, s);                     \
-  a += e;                                 \
-  c  = rotl32 (c, 10u);                   \
-}
-
-#define ROTATE_LEFT_WORKAROUND_BUG(a,n) ((a << n) | (a >> (32 - n)))
-
-#define RIPEMD160_STEP_WORKAROUND_BUG(f,a,b,c,d,e,x,K,s)  \
-{                                         \
-  a += K;                                 \
-  a += x;                                 \
-  a += f (b, c, d);                       \
-  a  = ROTATE_LEFT_WORKAROUND_BUG (a, s); \
-  a += e;                                 \
-  c  = rotl32 (c, 10u);                   \
-}
-
-#endif

diff --git a/include/kernel_vendor.h b/include/kernel_vendor.h
deleted file mode 100644 (file)
index 3adde96..0000000
--- a/include/kernel_vendor.h
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- * Author......: Jens Steube <jens.steube@gmail.com>
- * License.....: MIT
- */
-
-//fails on intel opencl sdk
-//#pragma OPENCL EXTENSION cl_khr_int64_base_atomics     : enable
-//#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
-
-/**
- * device type
- */
-
-#define DEVICE_TYPE_CPU   2
-#define DEVICE_TYPE_GPU   4
-#define DEVICE_TYPE_ACCEL 8
-
-#if   DEVICE_TYPE == DEVICE_TYPE_CPU
-#define IS_CPU
-#elif DEVICE_TYPE == DEVICE_TYPE_GPU
-#define IS_GPU
-#elif DEVICE_TYPE == DEVICE_TYPE_ACCEL
-#define IS_ACCEL
-#endif
-
-/**
- * vendor specific
- */
-
-#if   VENDOR_ID == (1 << 0)
-#define IS_AMD
-//#define IS_GENERIC
-#elif VENDOR_ID == (1 << 1)
-#define IS_APPLE
-#define IS_GENERIC
-#elif VENDOR_ID == (1 << 2)
-#define IS_INTEL_BEIGNET
-#define IS_GENERIC
-#elif VENDOR_ID == (1 << 3)
-#define IS_INTEL_SDK
-#define IS_GENERIC
-#elif VENDOR_ID == (1 << 4)
-#define IS_MESA
-#define IS_GENERIC
-#elif VENDOR_ID == (1 << 5)
-#define IS_NV
-//#define IS_GENERIC
-#elif VENDOR_ID == (1 << 6)
-#define IS_POCL
-#define IS_GENERIC
-#else
-#define IS_GENERIC
-#endif
-
-/**
- * AMD specific
- */
-
-#ifdef IS_AMD
-#pragma OPENCL EXTENSION cl_amd_media_ops  : enable
-#pragma OPENCL EXTENSION cl_amd_media_ops2 : enable
-#endif
-
-/**
- * Unrolling is generally enabled, for all device types and hash modes
- * There's a few exception when it's better not to unroll
- */
-
-// Some algorithms run into too much register pressure due to loop unrolling
-
-#ifdef IS_NV
-#ifdef IS_GPU
-
-#if KERN_TYPE == 1500
-#undef _unroll
-#endif
-#if KERN_TYPE == 1800
-#undef _unroll
-#endif
-#if KERN_TYPE == 3000
-#undef _unroll
-#endif
-#if KERN_TYPE == 6221
-#undef _unroll
-#endif
-#if KERN_TYPE == 6222
-#undef _unroll
-#endif
-#if KERN_TYPE == 6223
-#undef _unroll
-#endif
-#if KERN_TYPE == 6500
-#undef _unroll
-#endif
-#if KERN_TYPE == 7100
-#undef _unroll
-#endif
-#if KERN_TYPE == 7400
-#undef _unroll
-#endif
-#if KERN_TYPE == 7900
-#undef _unroll
-#endif
-#if KERN_TYPE == 8200
-#undef _unroll
-#endif
-#if KERN_TYPE == 10400
-#undef _unroll
-#endif
-#if KERN_TYPE == 10500
-#undef _unroll
-#endif
-#if KERN_TYPE == 10700
-#undef _unroll
-#endif
-#if KERN_TYPE == 12300
-#undef _unroll
-#endif
-#if KERN_TYPE == 12400
-#undef _unroll
-#endif
-#if KERN_TYPE == 13721
-#undef _unroll
-#endif
-#if KERN_TYPE == 13722
-#undef _unroll
-#endif
-#if KERN_TYPE == 13723
-#undef _unroll
-#endif
-
-#endif
-#endif
-
-#ifdef IS_AMD
-#ifdef IS_GPU
-
-#if KERN_TYPE == 3200
-#undef _unroll
-#endif
-#if KERN_TYPE == 5200
-#undef _unroll
-#endif
-#if KERN_TYPE == 6100
-#undef _unroll
-#endif
-#if KERN_TYPE == 6221
-#undef _unroll
-#endif
-#if KERN_TYPE == 6222
-#undef _unroll
-#endif
-#if KERN_TYPE == 6223
-#undef _unroll
-#endif
-#if KERN_TYPE == 6400
-#undef _unroll
-#endif
-#if KERN_TYPE == 6500
-#undef _unroll
-#endif
-#if KERN_TYPE == 6800
-#undef _unroll
-#endif
-#if KERN_TYPE == 7100
-#undef _unroll
-#endif
-#if KERN_TYPE == 7400
-#undef _unroll
-#endif
-#if KERN_TYPE == 8000
-#undef _unroll
-#endif
-#if KERN_TYPE == 8200
-#undef _unroll
-#endif
-#if KERN_TYPE == 10900
-#undef _unroll
-#endif
-#if KERN_TYPE == 11600
-#undef _unroll
-#endif
-#if KERN_TYPE == 12300
-#undef _unroll
-#endif
-#if KERN_TYPE == 12800
-#undef _unroll
-#endif
-#if KERN_TYPE == 12900
-#undef _unroll
-#endif
-#if KERN_TYPE == 13000
-#undef _unroll
-#endif
-#if KERN_TYPE == 13721
-#undef _unroll
-#endif
-#if KERN_TYPE == 13722
-#undef _unroll
-#endif
-#if KERN_TYPE == 13723
-#undef _unroll
-#endif
-
-#endif
-#endif
-
-// Some algorithms break due to loop unrolling, it's unknown why, probably compiler bugs
-// Can overlap with above cases
-
-#ifdef IS_AMD
-#ifdef IS_GPU
-
-#if KERN_TYPE == 1750
-#undef _unroll
-#endif
-#if KERN_TYPE == 1760
-#undef _unroll
-#endif
-#if KERN_TYPE == 6500
-#undef _unroll
-#endif
-#if KERN_TYPE == 7100
-#undef _unroll
-#endif
-#if KERN_TYPE == 9600
-#undef _unroll
-#endif
-#if KERN_TYPE == 12200
-#undef _unroll
-#endif
-#if KERN_TYPE == 12300
-#undef _unroll
-#endif
-
-#endif
-#endif

diff --git a/include/rp_kernel.h b/include/rp_kernel.h
deleted file mode 100644 (file)
index 17849f2..0000000
--- a/include/rp_kernel.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Author......: Jens Steube <jens.steube@gmail.com>
- * License.....: MIT
- */
-
-#define RULE_OP_MANGLE_NOOP             ':'
-#define RULE_OP_MANGLE_LREST            'l'
-#define RULE_OP_MANGLE_UREST            'u'
-#define RULE_OP_MANGLE_LREST_UFIRST     'c'
-#define RULE_OP_MANGLE_UREST_LFIRST     'C'
-#define RULE_OP_MANGLE_TREST            't'
-#define RULE_OP_MANGLE_TOGGLE_AT        'T'
-#define RULE_OP_MANGLE_REVERSE          'r'
-#define RULE_OP_MANGLE_DUPEWORD         'd'
-#define RULE_OP_MANGLE_DUPEWORD_TIMES   'p'
-#define RULE_OP_MANGLE_REFLECT          'f'
-#define RULE_OP_MANGLE_ROTATE_LEFT      '{'
-#define RULE_OP_MANGLE_ROTATE_RIGHT     '}'
-#define RULE_OP_MANGLE_APPEND           '$'
-#define RULE_OP_MANGLE_PREPEND          '^'
-#define RULE_OP_MANGLE_DELETE_FIRST     '['
-#define RULE_OP_MANGLE_DELETE_LAST      ']'
-#define RULE_OP_MANGLE_DELETE_AT        'D'
-#define RULE_OP_MANGLE_EXTRACT          'x'
-#define RULE_OP_MANGLE_OMIT             'O'
-#define RULE_OP_MANGLE_INSERT           'i'
-#define RULE_OP_MANGLE_OVERSTRIKE       'o'
-#define RULE_OP_MANGLE_TRUNCATE_AT      '\''
-#define RULE_OP_MANGLE_REPLACE          's'
-#define RULE_OP_MANGLE_PURGECHAR        '@'
-#define RULE_OP_MANGLE_TOGGLECASE_REC   'a'
-#define RULE_OP_MANGLE_DUPECHAR_FIRST   'z'
-#define RULE_OP_MANGLE_DUPECHAR_LAST    'Z'
-#define RULE_OP_MANGLE_DUPECHAR_ALL     'q'
-
-#define RULE_OP_REJECT_LESS             '<'
-#define RULE_OP_REJECT_GREATER          '>'
-#define RULE_OP_REJECT_CONTAIN          '!'
-#define RULE_OP_REJECT_NOT_CONTAIN      '/'
-#define RULE_OP_REJECT_EQUAL_FIRST      '('
-#define RULE_OP_REJECT_EQUAL_LAST       ')'
-#define RULE_OP_REJECT_EQUAL_AT         '='
-#define RULE_OP_REJECT_CONTAINS         '%'
-
-/* hashcat only */
-#define RULE_OP_MANGLE_SWITCH_FIRST     'k'
-#define RULE_OP_MANGLE_SWITCH_LAST      'K'
-#define RULE_OP_MANGLE_SWITCH_AT        '*'
-#define RULE_OP_MANGLE_CHR_SHIFTL       'L'
-#define RULE_OP_MANGLE_CHR_SHIFTR       'R'
-#define RULE_OP_MANGLE_CHR_INCR         '+'
-#define RULE_OP_MANGLE_CHR_DECR         '-'
-#define RULE_OP_MANGLE_REPLACE_NP1      '.'
-#define RULE_OP_MANGLE_REPLACE_NM1      ','
-#define RULE_OP_MANGLE_DUPEBLOCK_FIRST  'y'
-#define RULE_OP_MANGLE_DUPEBLOCK_LAST   'Y'
-#define RULE_OP_MANGLE_TITLE            'E'

diff --git a/include/rp_kernel_on_cpu.h b/include/rp_kernel_on_cpu.h
index 5b89f62..a0fda1c 100644 (file)
--- a/include/rp_kernel_on_cpu.h
+++ b/include/rp_kernel_on_cpu.h
@@ -9,7 +9,7 @@
 #define swap_workaround(n) __builtin_bswap32(n)
 
 #include "common.h"
-#include "rp_kernel.h"
+#include "inc_rp.h"
 
 u32 apply_rule (const u32 name, const u32 p0, const u32 p1, u32 buf0[4], u32 buf1[4], const u32 in_len);
 u32 apply_rules (u32 *cmds, u32 buf0[4], u32 buf1[4], const u32 len);

diff --git a/include/shared.h b/include/shared.h
index d5b00ab..5b64b5b 100644 (file)
--- a/include/shared.h
+++ b/include/shared.h
@@ -9,8 +9,8 @@
 #ifndef SHARED_H
 #define SHARED_H
 
-#include <common.h>
-#include <constants.h>
+#include "common.h"
+#include "inc_hash_constants.h"
 
 /**
  * thread management
@@ -108,18 +108,18 @@ static inline int  CPU_ISSET (int num, cpu_set_t *cs) { return (cs->count & (1 <
 #define hc_sleep(x) sleep ((x));
 #endif
 
-#include <ext_OpenCL.h>
+#include "ext_OpenCL.h"
 
 /**
  * temperature management
  */
 
 #if _WIN
-#include <ext_ADL.h>
-#include <ext_nvapi.h>
+#include "ext_ADL.h"
+#include "ext_nvapi.h"
 #else
-#include <ext_ADL.h>
-#include <ext_nvml.h>
+#include "ext_ADL.h"
+#include "ext_nvml.h"
 #endif
 
 /**
@@ -184,9 +184,9 @@ typedef pthread_t         hc_thread_t;
 typedef pthread_mutex_t   hc_thread_mutex_t;
 #endif
 
-#include <types.h>
+#include "types.h"
 #include "rp_cpu.h"
-#include "rp_kernel.h"
+#include "inc_rp.h"
 
 /**
  * valid project specific global stuff

diff --git a/src/Makefile b/src/Makefile
index 7509900..1515e5a 100644 (file)
--- a/src/Makefile
+++ b/src/Makefile
@@ -87,7 +87,7 @@ VERSION_TAG              := $(shell test -d .git && git describe --tags --dirty=
 ## Compiler flags
 ##
 
-CFLAGS                   := -pipe -W -Wall -std=c99 -Iinclude/ -I$(OPENCL_HEADERS_KHRONOS)/
+CFLAGS                   := -pipe -W -Wall -std=c99 -Iinclude/ -IOpenCL/ -I$(OPENCL_HEADERS_KHRONOS)/
 
 ifndef DEBUG
 CFLAGS                   += -O2

diff --git a/src/hashcat.c b/src/hashcat.c
index e23aabd..417b60d 100644 (file)
--- a/src/hashcat.c
+++ b/src/hashcat.c
@@ -14556,7 +14556,7 @@ int main (int argc, char **argv)
 
       // we don't have sm_* on vendors not NV but it doesn't matter
 
-      snprintf (build_opts, sizeof (build_opts) - 1, "-cl-std=CL1.1 -I\"%s/\" -DVENDOR_ID=%u -DCUDA_ARCH=%d -DVECT_SIZE=%u -DDEVICE_TYPE=%u -DKERN_TYPE=%u -D_unroll", shared_dir, device_param->vendor_id, (device_param->sm_major * 100) + device_param->sm_minor, device_param->vector_width, (u32) device_param->device_type, kern_type);
+      snprintf (build_opts, sizeof (build_opts) - 1, "-cl-std=CL1.1 -I\"%s/OpenCL\" -DVENDOR_ID=%u -DCUDA_ARCH=%d -DVECT_SIZE=%u -DDEVICE_TYPE=%u -DKERN_TYPE=%u -D_unroll", shared_dir, device_param->vendor_id, (device_param->sm_major * 100) + device_param->sm_minor, device_param->vector_width, (u32) device_param->device_type, kern_type);
 
       if (device_param->vendor_id == VENDOR_ID_INTEL_SDK)
       {

diff --git a/tools/package_bin.sh b/tools/package_bin.sh
index f0220f1..67328a9 100755 (executable)
--- a/tools/package_bin.sh
+++ b/tools/package_bin.sh
@@ -12,7 +12,6 @@ rm -rf $OUT
 rm -rf $OUT.7z
 
 mkdir -p $OUT
-mkdir -p $OUT/include
 
 cp    $IN/hashcat??.exe                 $OUT/
 cp    $IN/hashcat??.bin                 $OUT/
@@ -27,11 +26,6 @@ cp -r $IN/extra                         $OUT/
 cp    $IN/example.dict                  $OUT/
 cp    $IN/example[0123456789]*.hash    $OUT/
 cp    $IN/example[0123456789]*.cmd     $OUT/
-
-cp -r $IN/include/constants.h          $OUT/include
-cp -r $IN/include/kernel_functions.c   $OUT/include
-cp -r $IN/include/kernel_vendor.h       $OUT/include
-cp -r $IN/include/rp_kernel.h           $OUT/include
 cp -r $IN/OpenCL                        $OUT/
 
 # since for the binary distribution we still use .bin, we need to rewrite the commands
@@ -54,7 +48,6 @@ unix2dos $OUT/rules/*.rule
 unix2dos $OUT/rules/hybrid/*.rule
 unix2dos $OUT/docs/*
 unix2dos $OUT/example*.cmd
-unix2dos $OUT/include/*
 unix2dos $OUT/OpenCL/*
 unix2dos $OUT/hashcat.hctune
 
@@ -72,7 +65,6 @@ chmod 755 $OUT/example*.sh
 chmod 755 $OUT/extra
 chmod 755 $OUT/extra/tab_completion/*.sh
 chmod 755 $OUT/extra/tab_completion/install
-chmod 755 $OUT/include
 chmod 644 $OUT/include/*
 chmod 755 $OUT/OpenCL
 chmod 644 $OUT/OpenCL/*