X-Git-Url: https://www.flypig.org.uk/git/?a=blobdiff_plain;f=src%2Fhashcat.c;h=41b61444158f509ad02efb838b70408309d18baa;hb=6be99f523c1f5b86818825923d04bc22080f43fb;hp=031ee99b5d60ee9b4bf4a7aed8faf6e0c172fb27;hpb=e5359036ff97ac81f8092bf4d4745d9d59e35aaf;p=hashcat.git

diff --git a/src/hashcat.c b/src/hashcat.c
index 031ee99..41b6144 100644
--- a/src/hashcat.c
+++ b/src/hashcat.c
@@ -6,7 +6,7 @@
  * License.....: MIT
  */
 
-#ifdef OSX
+#ifdef DARWIN
 #include <stdio.h>
 #endif
 
@@ -33,6 +33,7 @@ double TARGET_MS_PROFILE[4]     = { 2, 12, 96, 480 };
 #define MARKOV_DISABLE          0
 #define MARKOV_CLASSIC          0
 #define BENCHMARK               0
+#define STDOUT_FLAG             0
 #define RESTORE                 0
 #define RESTORE_TIMER           60
 #define RESTORE_DISABLE         0
@@ -74,9 +75,10 @@ double TARGET_MS_PROFILE[4]     = { 2, 12, 96, 480 };
 #define SEPARATOR               ':'
 #define BITMAP_MIN              16
 #define BITMAP_MAX              24
+#define NVIDIA_SPIN_DAMP        100
 #define GPU_TEMP_DISABLE        0
 #define GPU_TEMP_ABORT          90
-#define GPU_TEMP_RETAIN         0
+#define GPU_TEMP_RETAIN         75
 #define WORKLOAD_PROFILE        2
 #define KERNEL_ACCEL            0
 #define KERNEL_LOOPS            0
@@ -150,6 +152,8 @@ double TARGET_MS_PROFILE[4]     = { 2, 12, 96, 480 };
 
 #define NUM_DEFAULT_BENCHMARK_ALGORITHMS 143
 
+#define NVIDIA_100PERCENTCPU_WORKAROUND 100
+
 #define global_free(attr)       \
 {                               \
   myfree ((void *) data.attr);  \
@@ -164,6 +168,12 @@ double TARGET_MS_PROFILE[4]     = { 2, 12, 96, 480 };
   attr = NULL;            \
 }
 
+#if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__)
+#define HC_API_CALL __stdcall
+#else
+#define HC_API_CALL
+#endif
+
 static uint default_benchmark_algorithms[NUM_DEFAULT_BENCHMARK_ALGORITHMS] =
 {
   900,
@@ -352,7 +362,7 @@ const char *USAGE_BIG[] =
   "- [ Options ] -",
   "",
   " Options Short / Long          | Type | Description                                          | Example",
-  "===============================|======|======================================================|=======================",
+  "===============================+======+======================================================+=======================",
   " -m, --hash-type               | Num  | Hash-type, see references below                      | -m 1000",
   " -a, --attack-mode             | Num  | Attack-mode, see references below                    | -a 3",
   " -V, --version                 |      | Print version                                        |",
@@ -380,8 +390,9 @@ const char *USAGE_BIG[] =
   "     --outfile-autohex-disable |      | Disable the use of $HEX[] in output plains           |",
   "     --outfile-check-timer     | Num  | Sets seconds between outfile checks to X             | --outfile-check=30",
   " -p, --separator               | Char | Separator char for hashlists and outfile             | -p :",
-  "     --show                    |      | Show cracked passwords only                          |",
-  "     --left                    |      | Show un-cracked passwords only                       |",
+  "     --stdout                  |      | Do not crack a hash, instead print candidates only   |",
+  "     --show                    |      | Compare hashlist with potfile; Show cracked hashes   |",
+  "     --left                    |      | Compare hashlist with potfile; Show uncracked hashes |",
   "     --username                |      | Enable ignoring of usernames in hashfile             |",
   "     --remove                  |      | Enable remove of hash once it is cracked             |",
   "     --remove-timer            | Num  | Update input hash file each X seconds                | --remove-timer=30",
@@ -402,11 +413,12 @@ const char *USAGE_BIG[] =
   "     --cpu-affinity            | Str  | Locks to CPU devices, separate with comma            | --cpu-affinity=1,2,3",
   "     --opencl-platforms        | Str  | OpenCL platforms to use, separate with comma         | --opencl-platforms=2",
   " -d, --opencl-devices          | Str  | OpenCL devices to use, separate with comma           | -d 1",
-  "     --opencl-device-types     | Str  | OpenCL device-types to use, separate with comma      | --opencl-device-type=1",
+  " -D, --opencl-device-types     | Str  | OpenCL device-types to use, separate with comma      | -D 1",
   "     --opencl-vector-width     | Num  | Manual override OpenCL vector-width to X             | --opencl-vector=4",
   " -w, --workload-profile        | Num  | Enable a specific workload profile, see pool below   | -w 3",
   " -n, --kernel-accel            | Num  | Manual workload tuning, set outerloop step size to X | -n 64",
   " -u, --kernel-loops            | Num  | Manual workload tuning, set innerloop step size to X | -u 256",
+  "     --nvidia-spin-damp        | Num  | Workaround NVidias CPU burning loop bug, in percent  | --nvidia-spin-damp=50",
   "     --gpu-temp-disable        |      | Disable temperature and fanspeed reads and triggers  |",
   #ifdef HAVE_HWMON
   "     --gpu-temp-abort          | Num  | Abort if GPU temperature reaches X degrees celsius   | --gpu-temp-abort=100",
@@ -417,9 +429,9 @@ const char *USAGE_BIG[] =
   " -s, --skip                    | Num  | Skip X words from the start                          | -s 1000000",
   " -l, --limit                   | Num  | Limit X words from the start + skipped words         | -l 1000000",
   "     --keyspace                |      | Show keyspace base:mod values and quit               |",
-  " -j, --rule-left               | Rule | Single Rule applied to each word from left wordlist  | -j 'c'",
-  " -k, --rule-right              | Rule | Single Rule applied to each word from right wordlist | -k '^-'",
-  " -r, --rules-file              | File | Multiple Rules applied to each word from wordlists   | -r rules/best64.rule",
+  " -j, --rule-left               | Rule | Single rule applied to each word from left wordlist  | -j 'c'",
+  " -k, --rule-right              | Rule | Single rule applied to each word from right wordlist | -k '^-'",
+  " -r, --rules-file              | File | Multiple rules applied to each word from wordlists   | -r rules/best64.rule",
   " -g, --generate-rules          | Num  | Generate X random rules                              | -g 10000",
   "     --generate-rules-func-min | Num  | Force min X funcs per rule                           |",
   "     --generate-rules-func-max | Num  | Force max X funcs per rule                           |",
@@ -570,6 +582,7 @@ const char *USAGE_BIG[] =
   "     22 | Juniper Netscreen/SSG (ScreenOS)                 | Operating-Systems",
   "    501 | Juniper IVE                                      | Operating-Systems",
   "   5800 | Android PIN                                      | Operating-Systems",
+  "  13800 | Windows 8+ phone PIN/Password                    | Operating-Systems",
   "   8100 | Citrix Netscaler                                 | Operating-Systems",
   "   8500 | RACF                                             | Operating-Systems",
   "   7200 | GRUB 2                                           | Operating-Systems",
@@ -715,11 +728,20 @@ const char *USAGE_BIG[] =
   "  3 | High        |  96 ms  | High              | Unresponsive",
   "  4 | Nightmare   | 480 ms  | Insane            | Headless",
   "",
-  "If you have no idea what just happened then visit the following pages:",
+  "- [ Basic Examples ] -",
+  "",
+  "  Attack-          | Hash- |",
+  "  Mode             | Type  | Example command",
+  " ==================+=======+==================================================================",
+  "  Wordlist         | $P$   | %s -a 0 -m 400 example400.hash example.dict",
+  "  Wordlist + Rules | MD5   | %s -a 0 -m 0 example0.hash example.dict -r rules/best64.rule",
+  "  Brute-Force      | MD5   | %s -a 3 -m 0 example0.hash ?a?a?a?a?a?a",
+  "  Combinator       | MD5   | %s -a 1 -m 0 example0.hash example.dict example.dict",
+  "",
+  "If you still have no idea what just happened try following pages:",
   "",
   "* https://hashcat.net/wiki/#howtos_videos_papers_articles_etc_in_the_wild",
   "* https://hashcat.net/wiki/#frequently_asked_questions",
-  "",
   NULL
 };
 
@@ -894,15 +916,7 @@ void status_display_machine_readable ()
    * flush
    */
 
-  #ifdef _WIN
-  fputc ('\r', out);
-  fputc ('\n', out);
-  #endif
-
-  #ifdef _POSIX
-  fputc ('\n', out);
-  #endif
-
+  fputs (EOL, out);
   fflush (out);
 }
 
@@ -910,7 +924,6 @@ void status_display ()
 {
   if (data.devices_status == STATUS_INIT)     return;
   if (data.devices_status == STATUS_STARTING) return;
-  if (data.devices_status == STATUS_BYPASS)   return;
 
   if (data.machine_readable == 1)
   {
@@ -1531,6 +1544,12 @@ void status_display ()
   }
 
   #ifdef HAVE_HWMON
+
+  if (data.devices_status == STATUS_EXHAUSTED)  return;
+  if (data.devices_status == STATUS_CRACKED)    return;
+  if (data.devices_status == STATUS_ABORTED)    return;
+  if (data.devices_status == STATUS_QUIT)       return;
+
   if (data.gpu_temp_disable == 0)
   {
     hc_thread_mutex_lock (mux_adl);
@@ -1614,6 +1633,7 @@ void status_display ()
 
     hc_thread_mutex_unlock (mux_adl);
   }
+
   #endif // HAVE_HWMON
 }
 
@@ -1662,7 +1682,6 @@ static void status_benchmark ()
 {
   if (data.devices_status == STATUS_INIT)     return;
   if (data.devices_status == STATUS_STARTING) return;
-  if (data.devices_status == STATUS_BYPASS)   return;
 
   if (data.machine_readable == 1)
   {
@@ -1733,7 +1752,14 @@ static void status_benchmark ()
 
     format_speed_display (hashes_dev_ms[device_id] * 1000, display_dev_cur, sizeof (display_dev_cur));
 
-    log_info ("Speed.Dev.#%d.: %9sH/s (%0.2fms)", device_id + 1, display_dev_cur, exec_all_ms[device_id]);
+    if (data.devices_active >= 10)
+    {
+      log_info ("Speed.Dev.#%d: %9sH/s (%0.2fms)", device_id + 1, display_dev_cur, exec_all_ms[device_id]);
+    }
+    else
+    {
+      log_info ("Speed.Dev.#%d.: %9sH/s (%0.2fms)", device_id + 1, display_dev_cur, exec_all_ms[device_id]);
+    }
   }
 
   char display_all_cur[16] = { 0 };
@@ -1815,6 +1841,26 @@ static void generate_cached_kernel_amp_filename (const uint attack_kern, char *p
   snprintf (cached_file, 255, "%s/kernels/amp_a%d.%s.kernel", profile_dir, attack_kern, device_name_chksum);
 }
 
+static char *filename_from_filepath (char *filepath)
+{
+  char *ptr = NULL;
+
+  if ((ptr = strrchr (filepath, '/')) != NULL)
+  {
+    ptr++;
+  }
+  else if ((ptr = strrchr (filepath, '\\')) != NULL)
+  {
+    ptr++;
+  }
+  else
+  {
+    ptr = filepath;
+  }
+
+  return ptr;
+}
+
 static uint convert_from_hex (char *line_buf, const uint line_len)
 {
   if (line_len & 1) return (line_len); // not in hex
@@ -2229,14 +2275,14 @@ static void check_cracked (hc_device_param_t *device_param, const uint salt_pos)
 
     uint cpt_cracked = 0;
 
+    hc_thread_mutex_lock (mux_display);
+
     for (uint i = 0; i < num_cracked; i++)
     {
       const uint hash_pos = cracked[i].hash_pos;
 
       if (data.digests_shown[hash_pos] == 1) continue;
 
-      hc_thread_mutex_lock (mux_display);
-
       if ((data.opts_type & OPTS_TYPE_PT_NEVERCRACK) == 0)
       {
         data.digests_shown[hash_pos] = 1;
@@ -2257,11 +2303,11 @@ static void check_cracked (hc_device_param_t *device_param, const uint salt_pos)
 
       if (data.salts_done == data.salts_cnt) data.devices_status = STATUS_CRACKED;
 
-      hc_thread_mutex_unlock (mux_display);
-
       check_hash (device_param, &cracked[i]);
     }
 
+    hc_thread_mutex_unlock (mux_display);
+
     myfree (cracked);
 
     if (cpt_cracked > 0)
@@ -2297,6 +2343,214 @@ static void check_cracked (hc_device_param_t *device_param, const uint salt_pos)
   }
 }
 
+// stolen from princeprocessor ;)
+
+typedef struct
+{
+  FILE *fp;
+
+  char  buf[BUFSIZ];
+  int   len;
+
+} out_t;
+
+static void out_flush (out_t *out)
+{
+  fwrite (out->buf, 1, out->len, out->fp);
+
+  out->len = 0;
+}
+
+static void out_push (out_t *out, const u8 *pw_buf, const int pw_len)
+{
+  char *ptr = out->buf + out->len;
+
+  memcpy (ptr, pw_buf, pw_len);
+
+  ptr[pw_len] = '\n';
+
+  out->len += pw_len + 1;
+
+  if (out->len >= BUFSIZ - 100)
+  {
+    out_flush (out);
+  }
+}
+
+static void process_stdout (hc_device_param_t *device_param, const uint pws_cnt)
+{
+  out_t out;
+
+  out.fp  = stdout;
+  out.len = 0;
+
+  uint plain_buf[16] = { 0 };
+
+  u8 *plain_ptr = (u8 *) plain_buf;
+
+  uint plain_len = 0;
+
+  const uint il_cnt = device_param->kernel_params_buf32[30]; // ugly, i know
+
+  if (data.attack_mode == ATTACK_MODE_STRAIGHT)
+  {
+    pw_t pw;
+
+    for (uint gidvid = 0; gidvid < pws_cnt; gidvid++)
+    {
+      gidd_to_pw_t (device_param, gidvid, &pw);
+
+      const uint pos = device_param->innerloop_pos;
+
+      for (uint il_pos = 0; il_pos < il_cnt; il_pos++)
+      {
+        for (int i = 0; i < 8; i++)
+        {
+          plain_buf[i] = pw.i[i];
+        }
+
+        plain_len = pw.pw_len;
+
+        plain_len = apply_rules (data.kernel_rules_buf[pos + il_pos].cmds, &plain_buf[0], &plain_buf[4], plain_len);
+
+        if (plain_len > data.pw_max) plain_len = data.pw_max;
+
+        out_push (&out, plain_ptr, plain_len);
+      }
+    }
+  }
+  else if (data.attack_mode == ATTACK_MODE_COMBI)
+  {
+    pw_t pw;
+
+    for (uint gidvid = 0; gidvid < pws_cnt; gidvid++)
+    {
+      gidd_to_pw_t (device_param, gidvid, &pw);
+
+      for (uint il_pos = 0; il_pos < il_cnt; il_pos++)
+      {
+        for (int i = 0; i < 8; i++)
+        {
+          plain_buf[i] = pw.i[i];
+        }
+
+        plain_len = pw.pw_len;
+
+        char *comb_buf = (char *) device_param->combs_buf[il_pos].i;
+        uint  comb_len =          device_param->combs_buf[il_pos].pw_len;
+
+        if (data.combs_mode == COMBINATOR_MODE_BASE_LEFT)
+        {
+          memcpy (plain_ptr + plain_len, comb_buf, comb_len);
+        }
+        else
+        {
+          memmove (plain_ptr + comb_len, plain_ptr, plain_len);
+
+          memcpy (plain_ptr, comb_buf, comb_len);
+        }
+
+        plain_len += comb_len;
+
+        if (data.pw_max != PW_DICTMAX1)
+        {
+          if (plain_len > data.pw_max) plain_len = data.pw_max;
+        }
+
+        out_push (&out, plain_ptr, plain_len);
+      }
+    }
+  }
+  else if (data.attack_mode == ATTACK_MODE_BF)
+  {
+    for (uint gidvid = 0; gidvid < pws_cnt; gidvid++)
+    {
+      for (uint il_pos = 0; il_pos < il_cnt; il_pos++)
+      {
+        u64 l_off = device_param->kernel_params_mp_l_buf64[3] + gidvid;
+        u64 r_off = device_param->kernel_params_mp_r_buf64[3] + il_pos;
+
+        uint l_start = device_param->kernel_params_mp_l_buf32[5];
+        uint r_start = device_param->kernel_params_mp_r_buf32[5];
+
+        uint l_stop = device_param->kernel_params_mp_l_buf32[4];
+        uint r_stop = device_param->kernel_params_mp_r_buf32[4];
+
+        sp_exec (l_off, (char *) plain_ptr + l_start, data.root_css_buf, data.markov_css_buf, l_start, l_start + l_stop);
+        sp_exec (r_off, (char *) plain_ptr + r_start, data.root_css_buf, data.markov_css_buf, r_start, r_start + r_stop);
+
+        plain_len = data.css_cnt;
+
+        out_push (&out, plain_ptr, plain_len);
+      }
+    }
+  }
+  else if (data.attack_mode == ATTACK_MODE_HYBRID1)
+  {
+    pw_t pw;
+
+    for (uint gidvid = 0; gidvid < pws_cnt; gidvid++)
+    {
+      gidd_to_pw_t (device_param, gidvid, &pw);
+
+      for (uint il_pos = 0; il_pos < il_cnt; il_pos++)
+      {
+        for (int i = 0; i < 8; i++)
+        {
+          plain_buf[i] = pw.i[i];
+        }
+
+        plain_len = pw.pw_len;
+
+        u64 off = device_param->kernel_params_mp_buf64[3] + il_pos;
+
+        uint start = 0;
+        uint stop  = device_param->kernel_params_mp_buf32[4];
+
+        sp_exec (off, (char *) plain_ptr + plain_len, data.root_css_buf, data.markov_css_buf, start, start + stop);
+
+        plain_len += start + stop;
+
+        out_push (&out, plain_ptr, plain_len);
+      }
+    }
+  }
+  else if (data.attack_mode == ATTACK_MODE_HYBRID2)
+  {
+    pw_t pw;
+
+    for (uint gidvid = 0; gidvid < pws_cnt; gidvid++)
+    {
+      gidd_to_pw_t (device_param, gidvid, &pw);
+
+      for (uint il_pos = 0; il_pos < il_cnt; il_pos++)
+      {
+        for (int i = 0; i < 8; i++)
+        {
+          plain_buf[i] = pw.i[i];
+        }
+
+        plain_len = pw.pw_len;
+
+        u64 off = device_param->kernel_params_mp_buf64[3] + il_pos;
+
+        uint start = 0;
+        uint stop  = device_param->kernel_params_mp_buf32[4];
+
+        memmove (plain_ptr + stop, plain_ptr, plain_len);
+
+        sp_exec (off, (char *) plain_ptr, data.root_css_buf, data.markov_css_buf, start, start + stop);
+
+        plain_len += start + stop;
+
+        out_push (&out, plain_ptr, plain_len);
+      }
+    }
+  }
+
+  out_flush (&out);
+}
+
 static void save_hash ()
 {
   char *hashfile = data.hashfile;
@@ -2334,8 +2588,6 @@ static void save_hash ()
 
       if (data.hash_mode != 2500)
       {
-        char out_buf[HCBUFSIZ] = { 0 };
-
         if (data.username == 1)
         {
           user_t *user = data.hash_info[idx]->user;
@@ -2347,11 +2599,15 @@ static void save_hash ()
           fputc (separator, fp);
         }
 
+        char out_buf[HCBUFSIZ]; // scratch buffer
+
+        out_buf[0] = 0;
+
         ascii_digest (out_buf, salt_pos, digest_pos);
 
         fputs (out_buf, fp);
 
-        log_out (fp, "");
+        fputc ('\n', fp);
       }
       else
       {
@@ -2389,48 +2645,12 @@ static void save_hash ()
   unlink (old_hashfile);
 }
 
-static float find_kernel_power_div (const u64 total_left, const uint kernel_power_all)
-{
-  // function called only in case kernel_power_all > words_left
-
-  float kernel_power_div = (float) (total_left) / kernel_power_all;
-
-  kernel_power_div += kernel_power_div / 100;
-
-  u32 kernel_power_new = (u32) (kernel_power_all * kernel_power_div);
-
-  while (kernel_power_new < total_left)
-  {
-    kernel_power_div += kernel_power_div / 100;
-
-    kernel_power_new = (u32) (kernel_power_all * kernel_power_div);
-  }
-
-  if (data.quiet == 0)
-  {
-    clear_prompt ();
-
-    //log_info ("");
-
-    log_info ("INFO: approaching final keyspace, workload adjusted");
-    log_info ("");
-
-    fprintf (stdout, "%s", PROMPT);
-
-    fflush (stdout);
-  }
-
-  //if ((kernel_power_all * kernel_power_div) < 8) return 1;
-
-  return kernel_power_div;
-}
-
-static void run_kernel (const uint kern_run, hc_device_param_t *device_param, const uint num, const uint event_update)
+static void run_kernel (const uint kern_run, hc_device_param_t *device_param, const uint num, const uint event_update, const uint iteration)
 {
   uint num_elements = num;
 
-  device_param->kernel_params_buf32[30] = data.combs_mode;
-  device_param->kernel_params_buf32[31] = num;
+  device_param->kernel_params_buf32[33] = data.combs_mode;
+  device_param->kernel_params_buf32[34] = num;
 
   uint kernel_threads = device_param->kernel_threads;
 
@@ -2447,9 +2667,6 @@ static void run_kernel (const uint kern_run, hc_device_param_t *device_param, co
     case KERN_RUN_3:    kernel = device_param->kernel3;     break;
   }
 
-  hc_clSetKernelArg (data.ocl, kernel, 21, sizeof (cl_uint), device_param->kernel_params[21]);
-  hc_clSetKernelArg (data.ocl, kernel, 22, sizeof (cl_uint), device_param->kernel_params[22]);
-  hc_clSetKernelArg (data.ocl, kernel, 23, sizeof (cl_uint), device_param->kernel_params[23]);
   hc_clSetKernelArg (data.ocl, kernel, 24, sizeof (cl_uint), device_param->kernel_params[24]);
   hc_clSetKernelArg (data.ocl, kernel, 25, sizeof (cl_uint), device_param->kernel_params[25]);
   hc_clSetKernelArg (data.ocl, kernel, 26, sizeof (cl_uint), device_param->kernel_params[26]);
@@ -2458,6 +2675,9 @@ static void run_kernel (const uint kern_run, hc_device_param_t *device_param, co
   hc_clSetKernelArg (data.ocl, kernel, 29, sizeof (cl_uint), device_param->kernel_params[29]);
   hc_clSetKernelArg (data.ocl, kernel, 30, sizeof (cl_uint), device_param->kernel_params[30]);
   hc_clSetKernelArg (data.ocl, kernel, 31, sizeof (cl_uint), device_param->kernel_params[31]);
+  hc_clSetKernelArg (data.ocl, kernel, 32, sizeof (cl_uint), device_param->kernel_params[32]);
+  hc_clSetKernelArg (data.ocl, kernel, 33, sizeof (cl_uint), device_param->kernel_params[33]);
+  hc_clSetKernelArg (data.ocl, kernel, 34, sizeof (cl_uint), device_param->kernel_params[34]);
 
   cl_event event;
 
@@ -2488,21 +2708,50 @@ static void run_kernel (const uint kern_run, hc_device_param_t *device_param, co
 
   hc_clFlush (data.ocl, device_param->command_queue);
 
+  if (device_param->nvidia_spin_damp)
+  {
+    if (data.devices_status == STATUS_RUNNING)
+    {
+      if (iteration < EXPECTED_ITERATIONS)
+      {
+        switch (kern_run)
+        {
+          case KERN_RUN_1: if (device_param->exec_us_prev1[iteration]) usleep (device_param->exec_us_prev1[iteration] * device_param->nvidia_spin_damp); break;
+          case KERN_RUN_2: if (device_param->exec_us_prev2[iteration]) usleep (device_param->exec_us_prev2[iteration] * device_param->nvidia_spin_damp); break;
+          case KERN_RUN_3: if (device_param->exec_us_prev3[iteration]) usleep (device_param->exec_us_prev3[iteration] * device_param->nvidia_spin_damp); break;
+        }
+      }
+    }
+  }
+
   hc_clWaitForEvents (data.ocl, 1, &event);
 
-  if (event_update)
-  {
-    cl_ulong time_start;
-    cl_ulong time_end;
+  cl_ulong time_start;
+  cl_ulong time_end;
 
-    hc_clGetEventProfilingInfo (data.ocl, event, CL_PROFILING_COMMAND_START, sizeof (time_start), &time_start, NULL);
-    hc_clGetEventProfilingInfo (data.ocl, event, CL_PROFILING_COMMAND_END,   sizeof (time_end),   &time_end,   NULL);
+  hc_clGetEventProfilingInfo (data.ocl, event, CL_PROFILING_COMMAND_START, sizeof (time_start), &time_start, NULL);
+  hc_clGetEventProfilingInfo (data.ocl, event, CL_PROFILING_COMMAND_END,   sizeof (time_end),   &time_end,   NULL);
 
-    const double exec_time = (double) (time_end - time_start) / 1000000.0;
+  const double exec_us = (double) (time_end - time_start) / 1000;
 
+  if (data.devices_status == STATUS_RUNNING)
+  {
+    if (iteration < EXPECTED_ITERATIONS)
+    {
+      switch (kern_run)
+      {
+        case KERN_RUN_1: device_param->exec_us_prev1[iteration] = exec_us; break;
+        case KERN_RUN_2: device_param->exec_us_prev2[iteration] = exec_us; break;
+        case KERN_RUN_3: device_param->exec_us_prev3[iteration] = exec_us; break;
+      }
+    }
+  }
+
+  if (event_update)
+  {
     uint exec_pos = device_param->exec_pos;
 
-    device_param->exec_ms[exec_pos] = exec_time;
+    device_param->exec_ms[exec_pos] = exec_us / 1000;
 
     exec_pos++;
 
@@ -2715,8 +2964,15 @@ static void run_kernel_bzero (hc_device_param_t *device_param, cl_mem buf, const
   */
 }
 
-static void choose_kernel (hc_device_param_t *device_param, const uint attack_exec, const uint attack_mode, const uint opts_type, const salt_t *salt_buf, const uint highest_pw_len, const uint pws_cnt)
+static void choose_kernel (hc_device_param_t *device_param, const uint attack_exec, const uint attack_mode, const uint opts_type, const salt_t *salt_buf, const uint highest_pw_len, const uint pws_cnt, const uint fast_iteration)
 {
+  if (data.hash_mode == 2000)
+  {
+    process_stdout (device_param, pws_cnt);
+
+    return;
+  }
+
   if (attack_exec == ATTACK_EXEC_INSIDE_KERNEL)
   {
     if (attack_mode == ATTACK_MODE_BF)
@@ -2735,46 +2991,53 @@ static void choose_kernel (hc_device_param_t *device_param, const uint attack_ex
 
     if (highest_pw_len < 16)
     {
-      run_kernel (KERN_RUN_1, device_param, pws_cnt, true);
+      run_kernel (KERN_RUN_1, device_param, pws_cnt, true, fast_iteration);
     }
     else if (highest_pw_len < 32)
     {
-      run_kernel (KERN_RUN_2, device_param, pws_cnt, true);
+      run_kernel (KERN_RUN_2, device_param, pws_cnt, true, fast_iteration);
     }
     else
     {
-      run_kernel (KERN_RUN_3, device_param, pws_cnt, true);
+      run_kernel (KERN_RUN_3, device_param, pws_cnt, true, fast_iteration);
     }
   }
   else
   {
     run_kernel_amp (device_param, pws_cnt);
 
-    run_kernel (KERN_RUN_1, device_param, pws_cnt, false);
+    run_kernel (KERN_RUN_1, device_param, pws_cnt, false, 0);
 
     if (opts_type & OPTS_TYPE_HOOK12)
     {
-      run_kernel (KERN_RUN_12, device_param, pws_cnt, false);
+      run_kernel (KERN_RUN_12, device_param, pws_cnt, false, 0);
+
+      hc_clEnqueueReadBuffer (data.ocl, device_param->command_queue, device_param->d_hooks, CL_TRUE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL);
+
+      // do something with data
+
+      hc_clEnqueueWriteBuffer (data.ocl, device_param->command_queue, device_param->d_hooks, CL_TRUE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL);
     }
 
     uint iter = salt_buf->salt_iter;
 
     uint loop_step = device_param->kernel_loops;
 
-    for (uint loop_pos = 0; loop_pos < iter; loop_pos += loop_step)
+    for (uint loop_pos = 0, slow_iteration = 0; loop_pos < iter; loop_pos += loop_step, slow_iteration++)
     {
       uint loop_left = iter - loop_pos;
 
       loop_left = MIN (loop_left, loop_step);
 
-      device_param->kernel_params_buf32[25] = loop_pos;
-      device_param->kernel_params_buf32[26] = loop_left;
+      device_param->kernel_params_buf32[28] = loop_pos;
+      device_param->kernel_params_buf32[29] = loop_left;
 
-      run_kernel (KERN_RUN_2, device_param, pws_cnt, true);
+      run_kernel (KERN_RUN_2, device_param, pws_cnt, true, slow_iteration);
 
       if (data.devices_status == STATUS_CRACKED) break;
       if (data.devices_status == STATUS_ABORTED) break;
       if (data.devices_status == STATUS_QUIT)    break;
+      if (data.devices_status == STATUS_BYPASS)  break;
 
       /**
        * speed
@@ -2802,7 +3065,7 @@ static void choose_kernel (hc_device_param_t *device_param, const uint attack_ex
 
     if (opts_type & OPTS_TYPE_HOOK23)
     {
-      run_kernel (KERN_RUN_23, device_param, pws_cnt, false);
+      run_kernel (KERN_RUN_23, device_param, pws_cnt, false, 0);
 
       hc_clEnqueueReadBuffer (data.ocl, device_param->command_queue, device_param->d_hooks, CL_TRUE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL);
 
@@ -2811,7 +3074,7 @@ static void choose_kernel (hc_device_param_t *device_param, const uint attack_ex
       hc_clEnqueueWriteBuffer (data.ocl, device_param->command_queue, device_param->d_hooks, CL_TRUE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL);
     }
 
-    run_kernel (KERN_RUN_3, device_param, pws_cnt, false);
+    run_kernel (KERN_RUN_3, device_param, pws_cnt, false, 0);
   }
 }
 
@@ -2907,17 +3170,17 @@ static double try_run (hc_device_param_t *device_param, const u32 kernel_accel,
 {
   const u32 kernel_power_try = device_param->device_processors * device_param->kernel_threads * kernel_accel;
 
-  device_param->kernel_params_buf32[25] = 0;
-  device_param->kernel_params_buf32[26] = kernel_loops; // not a bug, both need to be set
-  device_param->kernel_params_buf32[27] = kernel_loops; // because there's two variables for inner iters for slow and fast hashes
+  device_param->kernel_params_buf32[28] = 0;
+  device_param->kernel_params_buf32[29] = kernel_loops; // not a bug, both need to be set
+  device_param->kernel_params_buf32[30] = kernel_loops; // because there's two variables for inner iters for slow and fast hashes
 
   if (data.attack_exec == ATTACK_EXEC_INSIDE_KERNEL)
   {
-    run_kernel (KERN_RUN_1, device_param, kernel_power_try, true);
+    run_kernel (KERN_RUN_1, device_param, kernel_power_try, true, 0);
   }
   else
   {
-    run_kernel (KERN_RUN_2, device_param, kernel_power_try, true);
+    run_kernel (KERN_RUN_2, device_param, kernel_power_try, true, 0);
   }
 
   const double exec_ms_prev = get_avg_exec_time (device_param, 1);
@@ -2944,10 +3207,13 @@ static void autotune (hc_device_param_t *device_param)
 
   if ((kernel_loops_min == kernel_loops_max) && (kernel_accel_min == kernel_accel_max))
   {
-    try_run (device_param, kernel_accel, kernel_loops);
-    try_run (device_param, kernel_accel, kernel_loops);
-    try_run (device_param, kernel_accel, kernel_loops);
-    try_run (device_param, kernel_accel, kernel_loops);
+    if (data.hash_mode != 2000)
+    {
+      try_run (device_param, kernel_accel, kernel_loops);
+      try_run (device_param, kernel_accel, kernel_loops);
+      try_run (device_param, kernel_accel, kernel_loops);
+      try_run (device_param, kernel_accel, kernel_loops);
+    }
 
     device_param->kernel_accel = kernel_accel;
     device_param->kernel_loops = kernel_loops;
@@ -3133,6 +3399,10 @@ static void autotune (hc_device_param_t *device_param)
 
   memset (device_param->exec_ms, 0, EXEC_CACHE * sizeof (double));
 
+  memset (device_param->exec_us_prev1, 0, EXPECTED_ITERATIONS * sizeof (double));
+  memset (device_param->exec_us_prev2, 0, EXPECTED_ITERATIONS * sizeof (double));
+  memset (device_param->exec_us_prev3, 0, EXPECTED_ITERATIONS * sizeof (double));
+
   // store
 
   device_param->kernel_accel = kernel_accel;
@@ -3148,8 +3418,8 @@ static void autotune (hc_device_param_t *device_param)
   {
     clear_prompt ();
 
-    log_info ("Device #%u: autotuned kernel-accel to %u\n"
-              "Device #%u: autotuned kernel-loops to %u\n",
+    log_info ("- Device #%u: autotuned kernel-accel to %u\n"
+              "- Device #%u: autotuned kernel-loops to %u\n",
               device_param->device_id + 1, kernel_accel,
               device_param->device_id + 1, kernel_loops);
 
@@ -3226,9 +3496,9 @@ static void run_cracker (hc_device_param_t *device_param, const uint pws_cnt)
 
     salt_t *salt_buf = &data.salts_buf[salt_pos];
 
-    device_param->kernel_params_buf32[24] = salt_pos;
-    device_param->kernel_params_buf32[28] = salt_buf->digests_cnt;
-    device_param->kernel_params_buf32[29] = salt_buf->digests_offset;
+    device_param->kernel_params_buf32[27] = salt_pos;
+    device_param->kernel_params_buf32[31] = salt_buf->digests_cnt;
+    device_param->kernel_params_buf32[32] = salt_buf->digests_offset;
 
     FILE *combs_fp = device_param->combs_fp;
 
@@ -3250,14 +3520,21 @@ static void run_cracker (hc_device_param_t *device_param, const uint pws_cnt)
       if (data.devices_status == STATUS_QUIT)    break;
       if (data.devices_status == STATUS_BYPASS)  break;
 
+      uint fast_iteration = 0;
+
       uint innerloop_left = innerloop_cnt - innerloop_pos;
 
-      if (innerloop_left > innerloop_step) innerloop_left = innerloop_step;
+      if (innerloop_left > innerloop_step)
+      {
+        innerloop_left = innerloop_step;
+
+        fast_iteration = 1;
+      }
 
       device_param->innerloop_pos  = innerloop_pos;
       device_param->innerloop_left = innerloop_left;
 
-      device_param->kernel_params_buf32[27] = innerloop_left;
+      device_param->kernel_params_buf32[30] = innerloop_left;
 
       // i think we can get rid of this
       if (innerloop_left == 0)
@@ -3410,19 +3687,23 @@ static void run_cracker (hc_device_param_t *device_param, const uint pws_cnt)
         hc_timer_set (&device_param->timer_speed);
       }
 
-      choose_kernel (device_param, data.attack_exec, data.attack_mode, data.opts_type, salt_buf, highest_pw_len, pws_cnt);
+      choose_kernel (device_param, data.attack_exec, data.attack_mode, data.opts_type, salt_buf, highest_pw_len, pws_cnt, fast_iteration);
 
       if (data.devices_status == STATUS_STOP_AT_CHECKPOINT) check_checkpoint ();
 
       if (data.devices_status == STATUS_CRACKED) break;
       if (data.devices_status == STATUS_ABORTED) break;
       if (data.devices_status == STATUS_QUIT)    break;
+      if (data.devices_status == STATUS_BYPASS)  break;
 
       /**
        * result
        */
 
-      check_cracked (device_param, salt_pos);
+      if (data.benchmark == 0)
+      {
+        check_cracked (device_param, salt_pos);
+      }
 
       /**
        * progress
@@ -3816,7 +4097,9 @@ static void *thread_monitor (void *p)
   uint status_left  = data.status_timer;
 
   #ifdef HAVE_HWMON
-  uint hwmon_check   = 0;
+  uint hwmon_check = 0;
+
+  int slowdown_warnings = 0;
 
   // these variables are mainly used for fan control
 
@@ -3874,7 +4157,7 @@ static void *thread_monitor (void *p)
     return (p);
   }
 
-  while ((data.devices_status != STATUS_EXHAUSTED) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
+  while (data.shutdown_inner == 0)
   {
     hc_sleep (sleep_time);
 
@@ -3886,34 +4169,90 @@ static void *thread_monitor (void *p)
     {
       hc_thread_mutex_lock (mux_adl);
 
-      time_t temp_check_time;
-
-      time (&temp_check_time);
-
-      uint Ta = temp_check_time - last_temp_check_time; // set Ta = sleep_time; is not good enough (see --remove etc)
-
-      if (Ta == 0) Ta = 1;
-
       for (uint device_id = 0; device_id < data.devices_cnt; device_id++)
       {
         hc_device_param_t *device_param = &data.devices_param[device_id];
 
         if (device_param->skipped) continue;
 
-        if ((data.devices_param[device_id].device_type & CL_DEVICE_TYPE_GPU) == 0) continue;
+        if (device_param->device_vendor_id == VENDOR_ID_NV)
+        {
+          if (data.hm_nvapi)
+          {
+            NV_GPU_PERF_POLICIES_INFO_PARAMS_V1   perfPolicies_info   = { 0 };
+            NV_GPU_PERF_POLICIES_STATUS_PARAMS_V1 perfPolicies_status = { 0 };
 
-        const int temperature = hm_get_temperature_with_device_id (device_id);
+            perfPolicies_info.version   = MAKE_NVAPI_VERSION (NV_GPU_PERF_POLICIES_INFO_PARAMS_V1, 1);
+            perfPolicies_status.version = MAKE_NVAPI_VERSION (NV_GPU_PERF_POLICIES_STATUS_PARAMS_V1, 1);
 
-        if (temperature > (int) data.gpu_temp_abort)
-        {
-          log_error ("ERROR: Temperature limit on GPU %d reached, aborting...", device_id + 1);
+            hm_NvAPI_GPU_GetPerfPoliciesInfo (data.hm_nvapi, data.hm_device[device_id].nvapi, &perfPolicies_info);
 
-          if (data.devices_status != STATUS_QUIT) myabort ();
+            perfPolicies_status.info_value = perfPolicies_info.info_value;
 
-          break;
-        }
+            hm_NvAPI_GPU_GetPerfPoliciesStatus (data.hm_nvapi, data.hm_device[device_id].nvapi, &perfPolicies_status);
 
-        const int gpu_temp_retain = data.gpu_temp_retain;
+            if (perfPolicies_status.throttle & 2)
+            {
+              if (slowdown_warnings < 3)
+              {
+                if (data.quiet == 0) clear_prompt ();
+
+                log_info ("WARNING: Drivers temperature threshold hit on GPU #%d, expect performance to drop...", device_id + 1);
+
+                if (slowdown_warnings == 2)
+                {
+                  log_info ("");
+                }
+
+                if (data.quiet == 0) fprintf (stdout, "%s", PROMPT);
+                if (data.quiet == 0) fflush (stdout);
+
+                slowdown_warnings++;
+              }
+            }
+            else
+            {
+              slowdown_warnings = 0;
+            }
+          }
+        }
+      }
+
+      hc_thread_mutex_unlock (mux_adl);
+    }
+
+    if (hwmon_check == 1)
+    {
+      hc_thread_mutex_lock (mux_adl);
+
+      time_t temp_check_time;
+
+      time (&temp_check_time);
+
+      uint Ta = temp_check_time - last_temp_check_time; // set Ta = sleep_time; is not good enough (see --remove etc)
+
+      if (Ta == 0) Ta = 1;
+
+      for (uint device_id = 0; device_id < data.devices_cnt; device_id++)
+      {
+        hc_device_param_t *device_param = &data.devices_param[device_id];
+
+        if (device_param->skipped) continue;
+
+        if ((data.devices_param[device_id].device_type & CL_DEVICE_TYPE_GPU) == 0) continue;
+
+        const int temperature = hm_get_temperature_with_device_id (device_id);
+
+        if (temperature > (int) data.gpu_temp_abort)
+        {
+          log_error ("ERROR: Temperature limit on GPU %d reached, aborting...", device_id + 1);
+
+          if (data.devices_status != STATUS_QUIT) myabort ();
+
+          break;
+        }
+
+        const int gpu_temp_retain = data.gpu_temp_retain;
 
         if (gpu_temp_retain)
         {
@@ -3963,7 +4302,13 @@ static void *thread_monitor (void *p)
                   }
                   else if (device_param->device_vendor_id == VENDOR_ID_NV)
                   {
+                    #ifdef WIN
+                    hm_set_fanspeed_with_device_id_nvapi (device_id, fan_speed_new, 1);
+                    #endif
 
+                    #ifdef LINUX
+                    hm_set_fanspeed_with_device_id_xnvctrl (device_id, fan_speed_new);
+                    #endif
                   }
 
                   fan_speed_chgd[device_id] = 1;
@@ -3998,7 +4343,7 @@ static void *thread_monitor (void *p)
 
       time (&runtime_cur);
 
-      int runtime_left = data.runtime_start + data.runtime - runtime_cur;
+      int runtime_left = data.proc_start + data.runtime - runtime_cur;
 
       if (runtime_left <= 0)
       {
@@ -4034,7 +4379,7 @@ static void *thread_monitor (void *p)
 
       if (status_left == 0)
       {
-        //hc_thread_mutex_lock (mux_display);
+        hc_thread_mutex_lock (mux_display);
 
         if (data.quiet == 0) clear_prompt ();
 
@@ -4044,7 +4389,7 @@ static void *thread_monitor (void *p)
 
         if (data.quiet == 0) log_info ("");
 
-        //hc_thread_mutex_unlock (mux_display);
+        hc_thread_mutex_unlock (mux_display);
 
         status_left = data.status_timer;
       }
@@ -4101,7 +4446,7 @@ static void *thread_outfile_remove (void *p)
 
   uint check_left = outfile_check_timer; // or 1 if we want to check it at startup
 
-  while ((data.devices_status != STATUS_EXHAUSTED) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
+  while (data.shutdown_inner == 0)
   {
     hc_sleep (1);
 
@@ -4385,14 +4730,43 @@ static void pw_add (hc_device_param_t *device_param, const u8 *pw_buf, const int
   //}
 }
 
-static u32 get_power (const u32 kernel_power)
+static void set_kernel_power_final (const u64 kernel_power_final)
+{
+  if (data.quiet == 0)
+  {
+    clear_prompt ();
+
+    //log_info ("");
+
+    log_info ("INFO: approaching final keyspace, workload adjusted");
+    log_info ("");
+
+    fprintf (stdout, "%s", PROMPT);
+
+    fflush (stdout);
+  }
+
+  data.kernel_power_final = kernel_power_final;
+}
+
+static u32 get_power (hc_device_param_t *device_param)
 {
-  if (data.kernel_power_div)
+  const u64 kernel_power_final = data.kernel_power_final;
+
+  if (kernel_power_final)
   {
-    return (float) kernel_power * data.kernel_power_div;
+    const double device_factor = (double) device_param->hardware_power / data.hardware_power_all;
+
+    const u64 words_left_device = CEIL ((double) kernel_power_final * device_factor);
+
+    // work should be at least the hardware power available without any accelerator
+
+    const u64 work = MAX (words_left_device, device_param->hardware_power);
+
+    return work;
   }
 
-  return kernel_power;
+  return device_param->kernel_power;
 }
 
 static uint get_work (hc_device_param_t *device_param, const u64 max)
@@ -4400,21 +4774,23 @@ static uint get_work (hc_device_param_t *device_param, const u64 max)
   hc_thread_mutex_lock (mux_dispatcher);
 
   const u64 words_cur  = data.words_cur;
-  const u64 words_base = (data.limit == 0) ? data.words_base : data.limit;
+  const u64 words_base = (data.limit == 0) ? data.words_base : MIN (data.limit, data.words_base);
 
   device_param->words_off = words_cur;
 
+  const u64 kernel_power_all = data.kernel_power_all;
+
   const u64 words_left = words_base - words_cur;
 
-  if (data.kernel_power_all > words_left)
+  if (words_left < kernel_power_all)
   {
-    if (data.kernel_power_div == 0)
+    if (data.kernel_power_final == 0)
     {
-      data.kernel_power_div = find_kernel_power_div (words_left, data.kernel_power_all);
+      set_kernel_power_final (words_left);
     }
   }
 
-  const u32 kernel_power = get_power (device_param->kernel_power);
+  const u32 kernel_power = get_power (device_param);
 
   uint work = MIN (words_left, kernel_power);
 
@@ -4448,7 +4824,7 @@ static void *thread_calc_stdin (void *p)
 
   const uint attack_kern = data.attack_kern;
 
-  while ((data.devices_status != STATUS_EXHAUSTED) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
+  while ((data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
   {
     hc_thread_mutex_lock (mux_dispatcher);
 
@@ -4461,7 +4837,7 @@ static void *thread_calc_stdin (void *p)
 
     uint words_cur = 0;
 
-    while (words_cur < get_power (device_param->kernel_power))
+    while (words_cur < device_param->kernel_power)
     {
       char *line_buf = fgets (buf, HCBUFSIZ - 1, stdin);
 
@@ -4495,6 +4871,8 @@ static void *thread_calc_stdin (void *p)
         continue;
       }
 
+      // hmm that's always the case, or?
+
       if (attack_kern == ATTACK_KERN_STRAIGHT)
       {
         if ((line_len < data.pw_min) || (line_len > data.pw_max))
@@ -4511,25 +4889,6 @@ static void *thread_calc_stdin (void *p)
           continue;
         }
       }
-      else if (attack_kern == ATTACK_KERN_COMBI)
-      {
-        // do not check if minimum restriction is satisfied (line_len >= data.pw_min) here
-        // since we still need to combine the plains
-
-        if (line_len > data.pw_max)
-        {
-          hc_thread_mutex_lock (mux_counter);
-
-          for (uint salt_pos = 0; salt_pos < data.salts_cnt; salt_pos++)
-          {
-            data.words_progress_rejected[salt_pos] += data.combs_cnt;
-          }
-
-          hc_thread_mutex_unlock (mux_counter);
-
-          continue;
-        }
-      }
 
       pw_add (device_param, (u8 *) line_buf, line_len);
 
@@ -4593,7 +4952,7 @@ static void *thread_calc (void *p)
 
   if (attack_mode == ATTACK_MODE_BF)
   {
-    while ((data.devices_status != STATUS_EXHAUSTED) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
+    while ((data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
     {
       const uint work = get_work (device_param, -1);
 
@@ -4705,7 +5064,7 @@ static void *thread_calc (void *p)
 
     u64 words_cur = 0;
 
-    while ((data.devices_status != STATUS_EXHAUSTED) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
+    while ((data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
     {
       u64 words_off = 0;
       u64 words_fin = 0;
@@ -4885,12 +5244,12 @@ static void weak_hash_check (hc_device_param_t *device_param, const uint salt_po
 
   salt_t *salt_buf = &data.salts_buf[salt_pos];
 
-  device_param->kernel_params_buf32[24] = salt_pos;
-  device_param->kernel_params_buf32[27] = 1;
-  device_param->kernel_params_buf32[28] = salt_buf->digests_cnt;
-  device_param->kernel_params_buf32[29] = salt_buf->digests_offset;
-  device_param->kernel_params_buf32[30] = 0;
-  device_param->kernel_params_buf32[31] = 1;
+  device_param->kernel_params_buf32[27] = salt_pos;
+  device_param->kernel_params_buf32[30] = 1;
+  device_param->kernel_params_buf32[31] = salt_buf->digests_cnt;
+  device_param->kernel_params_buf32[32] = salt_buf->digests_offset;
+  device_param->kernel_params_buf32[33] = 0;
+  device_param->kernel_params_buf32[34] = 1;
 
   char *dictfile_old = data.dictfile;
 
@@ -4908,11 +5267,11 @@ static void weak_hash_check (hc_device_param_t *device_param, const uint salt_po
 
   if (data.attack_exec == ATTACK_EXEC_INSIDE_KERNEL)
   {
-    run_kernel (KERN_RUN_1, device_param, 1, false);
+    run_kernel (KERN_RUN_1, device_param, 1, false, 0);
   }
   else
   {
-    run_kernel (KERN_RUN_1, device_param, 1, false);
+    run_kernel (KERN_RUN_1, device_param, 1, false, 0);
 
     uint loop_step = 16;
 
@@ -4924,13 +5283,13 @@ static void weak_hash_check (hc_device_param_t *device_param, const uint salt_po
 
       loop_left = MIN (loop_left, loop_step);
 
-      device_param->kernel_params_buf32[25] = loop_pos;
-      device_param->kernel_params_buf32[26] = loop_left;
+      device_param->kernel_params_buf32[28] = loop_pos;
+      device_param->kernel_params_buf32[29] = loop_left;
 
-      run_kernel (KERN_RUN_2, device_param, 1, false);
+      run_kernel (KERN_RUN_2, device_param, 1, false, 0);
     }
 
-    run_kernel (KERN_RUN_3, device_param, 1, false);
+    run_kernel (KERN_RUN_3, device_param, 1, false, 0);
   }
 
   /**
@@ -4943,14 +5302,14 @@ static void weak_hash_check (hc_device_param_t *device_param, const uint salt_po
    * cleanup
    */
 
-  device_param->kernel_params_buf32[24] = 0;
-  device_param->kernel_params_buf32[25] = 0;
-  device_param->kernel_params_buf32[26] = 0;
   device_param->kernel_params_buf32[27] = 0;
   device_param->kernel_params_buf32[28] = 0;
   device_param->kernel_params_buf32[29] = 0;
   device_param->kernel_params_buf32[30] = 0;
   device_param->kernel_params_buf32[31] = 0;
+  device_param->kernel_params_buf32[32] = 0;
+  device_param->kernel_params_buf32[33] = 0;
+  device_param->kernel_params_buf32[34] = 0;
 
   data.dictfile = dictfile_old;
 
@@ -5314,7 +5673,7 @@ static uint hlfmt_detect (FILE *fp, uint max_check)
 // wrapper around mymalloc for ADL
 
 #if defined(HAVE_HWMON)
-void *__stdcall ADL_Main_Memory_Alloc (const int iSize)
+void *HC_API_CALL ADL_Main_Memory_Alloc (const int iSize)
 {
   return mymalloc (iSize);
 }
@@ -5336,8 +5695,6 @@ static uint generate_bitmaps (const uint digests_cnt, const uint dgst_size, cons
 
   for (uint i = 0; i < digests_cnt; i++)
   {
-    if (data.digests_shown[i] == 1) continue; // can happen with potfile
-
     uint *digest_ptr = (uint *) digests_buf_ptr;
 
     digests_buf_ptr += dgst_size;
@@ -5372,7 +5729,7 @@ static uint generate_bitmaps (const uint digests_cnt, const uint dgst_size, cons
  * main
  */
 
-#ifdef _WIN
+#ifdef WIN
 void SetConsoleWindowSize (const int x)
 {
   HANDLE h = GetStdHandle (STD_OUTPUT_HANDLE);
@@ -5400,7 +5757,7 @@ void SetConsoleWindowSize (const int x)
 
 int main (int argc, char **argv)
 {
-  #ifdef _WIN
+  #ifdef WIN
   SetConsoleWindowSize (132);
   #endif
 
@@ -5469,6 +5826,7 @@ int main (int argc, char **argv)
   uint  version                   = VERSION;
   uint  quiet                     = QUIET;
   uint  benchmark                 = BENCHMARK;
+  uint  stdout_flag               = STDOUT_FLAG;
   uint  show                      = SHOW;
   uint  left                      = LEFT;
   uint  username                  = USERNAME;
@@ -5528,6 +5886,7 @@ int main (int argc, char **argv)
   uint  workload_profile          = WORKLOAD_PROFILE;
   uint  kernel_accel              = KERNEL_ACCEL;
   uint  kernel_loops              = KERNEL_LOOPS;
+  uint  nvidia_spin_damp          = NVIDIA_SPIN_DAMP;
   uint  gpu_temp_disable          = GPU_TEMP_DISABLE;
   #ifdef HAVE_HWMON
   uint  gpu_temp_abort            = GPU_TEMP_ABORT;
@@ -5566,6 +5925,7 @@ int main (int argc, char **argv)
   #define IDX_FORCE                     0xff08
   #define IDX_RUNTIME                   0xff09
   #define IDX_BENCHMARK                 'b'
+  #define IDX_STDOUT_FLAG               0xff77
   #define IDX_HASH_MODE                 'm'
   #define IDX_ATTACK_MODE               'a'
   #define IDX_RP_FILE                   'r'
@@ -5600,11 +5960,12 @@ int main (int argc, char **argv)
   #define IDX_CPU_AFFINITY              0xff25
   #define IDX_OPENCL_DEVICES            'd'
   #define IDX_OPENCL_PLATFORMS          0xff72
-  #define IDX_OPENCL_DEVICE_TYPES       0xff73
+  #define IDX_OPENCL_DEVICE_TYPES       'D'
   #define IDX_OPENCL_VECTOR_WIDTH       0xff74
   #define IDX_WORKLOAD_PROFILE          'w'
   #define IDX_KERNEL_ACCEL              'n'
   #define IDX_KERNEL_LOOPS              'u'
+  #define IDX_NVIDIA_SPIN_DAMP          0xff79
   #define IDX_GPU_TEMP_DISABLE          0xff29
   #define IDX_GPU_TEMP_ABORT            0xff30
   #define IDX_GPU_TEMP_RETAIN           0xff31
@@ -5623,7 +5984,7 @@ int main (int argc, char **argv)
   #define IDX_CUSTOM_CHARSET_3          '3'
   #define IDX_CUSTOM_CHARSET_4          '4'
 
-  char short_options[] = "hVvm:a:r:j:k:g:o:t:d:n:u:c:p:s:l:1:2:3:4:ibw:";
+  char short_options[] = "hVvm:a:r:j:k:g:o:t:d:D:n:u:c:p:s:l:1:2:3:4:ibw:";
 
   struct option long_options[] =
   {
@@ -5646,6 +6007,7 @@ int main (int argc, char **argv)
     {"outfile-check-dir",         required_argument, 0, IDX_OUTFILE_CHECK_DIR},
     {"force",                     no_argument,       0, IDX_FORCE},
     {"benchmark",                 no_argument,       0, IDX_BENCHMARK},
+    {"stdout",                    no_argument,       0, IDX_STDOUT_FLAG},
     {"restore",                   no_argument,       0, IDX_RESTORE},
     {"restore-disable",           no_argument,       0, IDX_RESTORE_DISABLE},
     {"status",                    no_argument,       0, IDX_STATUS},
@@ -5683,6 +6045,7 @@ int main (int argc, char **argv)
     {"workload-profile",          required_argument, 0, IDX_WORKLOAD_PROFILE},
     {"kernel-accel",              required_argument, 0, IDX_KERNEL_ACCEL},
     {"kernel-loops",              required_argument, 0, IDX_KERNEL_LOOPS},
+    {"nvidia-spin-damp",          required_argument, 0, IDX_NVIDIA_SPIN_DAMP},
     {"gpu-temp-disable",          no_argument,       0, IDX_GPU_TEMP_DISABLE},
     #ifdef HAVE_HWMON
     {"gpu-temp-abort",            required_argument, 0, IDX_GPU_TEMP_ABORT},
@@ -5902,6 +6265,7 @@ int main (int argc, char **argv)
   uint runtime_chgd             = 0;
   uint kernel_loops_chgd        = 0;
   uint kernel_accel_chgd        = 0;
+  uint nvidia_spin_damp_chgd    = 0;
   uint attack_mode_chgd         = 0;
   uint outfile_format_chgd      = 0;
   uint rp_gen_seed_chgd         = 0;
@@ -5942,6 +6306,7 @@ int main (int argc, char **argv)
       case IDX_LIMIT:                     limit                     = atoll (optarg); break;
       case IDX_KEYSPACE:                  keyspace                  = 1;              break;
       case IDX_BENCHMARK:                 benchmark                 = 1;              break;
+      case IDX_STDOUT_FLAG:               stdout_flag               = 1;              break;
       case IDX_RESTORE:                                                               break;
       case IDX_RESTORE_DISABLE:           restore_disable           = 1;              break;
       case IDX_STATUS:                    status                    = 1;              break;
@@ -5989,6 +6354,8 @@ int main (int argc, char **argv)
                                           kernel_accel_chgd         = 1;              break;
       case IDX_KERNEL_LOOPS:              kernel_loops              = atoi (optarg);
                                           kernel_loops_chgd         = 1;              break;
+      case IDX_NVIDIA_SPIN_DAMP:          nvidia_spin_damp          = atoi (optarg);
+                                          nvidia_spin_damp_chgd     = 1;              break;
       case IDX_GPU_TEMP_DISABLE:          gpu_temp_disable          = 1;              break;
       #ifdef HAVE_HWMON
       case IDX_GPU_TEMP_ABORT:            gpu_temp_abort            = atoi (optarg);  break;
@@ -6052,6 +6419,14 @@ int main (int argc, char **argv)
       log_info ("%s (%s) starting in restore-mode...", PROGNAME, VERSION_TAG);
       log_info ("");
     }
+    else if (stdout_flag == 1)
+    {
+      // do nothing
+    }
+    else if (keyspace == 1)
+    {
+      // do nothing
+    }
     else
     {
       log_info ("%s (%s) starting...", PROGNAME, VERSION_TAG);
@@ -6077,7 +6452,7 @@ int main (int argc, char **argv)
     return (-1);
   }
 
-  if (hash_mode_chgd && hash_mode > 13799) // just added to remove compiler warnings for hash_mode_chgd
+  if (hash_mode_chgd && hash_mode > 13800) // just added to remove compiler warnings for hash_mode_chgd
   {
     log_error ("ERROR: Invalid hash-type specified");
 
@@ -6127,7 +6502,7 @@ int main (int argc, char **argv)
     {
       if (outfile_format > 1)
       {
-        log_error ("ERROR: Mixing outfile-format > 1 is not allowed together with left parameter");
+        log_error ("ERROR: Mixing outfile-format > 1 with left parameter is not allowed");
 
         return (-1);
       }
@@ -6144,7 +6519,7 @@ int main (int argc, char **argv)
     {
       if ((outfile_format > 7) && (outfile_format < 16))
       {
-        log_error ("ERROR: Mixing outfile-format > 7 is not allowed together with show parameter");
+        log_error ("ERROR: Mixing outfile-format > 7 with show parameter is not allowed");
 
         return (-1);
       }
@@ -6174,21 +6549,21 @@ int main (int argc, char **argv)
 
   if ((increment == 1) && (attack_mode == ATTACK_MODE_STRAIGHT))
   {
-    log_error ("ERROR: increment is not allowed in attack-mode 0");
+    log_error ("ERROR: Increment is not allowed in attack-mode 0");
 
     return (-1);
   }
 
   if ((increment == 0) && (increment_min_chgd == 1))
   {
-    log_error ("ERROR: increment-min is only supported together with increment switch");
+    log_error ("ERROR: Increment-min is only supported combined with increment switch");
 
     return (-1);
   }
 
   if ((increment == 0) && (increment_max_chgd == 1))
   {
-    log_error ("ERROR: increment-max is only supported together with increment switch");
+    log_error ("ERROR: Increment-max is only supported combined with increment switch");
 
     return (-1);
   }
@@ -6221,8 +6596,8 @@ int main (int argc, char **argv)
   {
     if (force == 0)
     {
-      log_info ("The manual use of the option -n (or --kernel-accel) is outdated");
-      log_info ("Please consider using the option -w instead");
+      log_info ("The manual use of the -n option (or --kernel-accel) is outdated");
+      log_info ("Please consider using the -w option instead");
       log_info ("You can use --force to override this but do not post error reports if you do so");
       log_info ("");
 
@@ -6248,8 +6623,8 @@ int main (int argc, char **argv)
   {
     if (force == 0)
     {
-      log_info ("The manual use of the option -u (or --kernel-loops) is outdated");
-      log_info ("Please consider using the option -w instead");
+      log_info ("The manual use of the -u option (or --kernel-loops) is outdated");
+      log_info ("Please consider using the -w option instead");
       log_info ("You can use --force to override this but do not post error reports if you do so");
       log_info ("");
 
@@ -6315,8 +6690,32 @@ int main (int argc, char **argv)
     case ATTACK_MODE_HYBRID2:  attack_kern = ATTACK_KERN_COMBI;    break;
   }
 
-  if (benchmark == 0)
+  if (benchmark == 1)
+  {
+    if (myargv[optind] != 0)
+    {
+      log_error ("ERROR: Invalid argument for benchmark mode specified");
+
+      return (-1);
+    }
+
+    if (attack_mode_chgd == 1)
+    {
+      if (attack_mode != ATTACK_MODE_BF)
+      {
+        log_error ("ERROR: Only attack-mode 3 allowed in benchmark mode");
+
+        return (-1);
+      }
+    }
+  }
+  else
   {
+    if (stdout_flag == 1) // no hash here
+    {
+      optind--;
+    }
+
     if (keyspace == 1)
     {
       int num_additional_params = 1;
@@ -6374,25 +6773,6 @@ int main (int argc, char **argv)
       return (-1);
     }
   }
-  else
-  {
-    if (myargv[optind] != 0)
-    {
-      log_error ("ERROR: Invalid argument for benchmark mode specified");
-
-      return (-1);
-    }
-
-    if (attack_mode_chgd == 1)
-    {
-      if (attack_mode != ATTACK_MODE_BF)
-      {
-        log_error ("ERROR: Only attack-mode 3 allowed in benchmark mode");
-
-        return (-1);
-      }
-    }
-  }
 
   if (skip != 0 && limit != 0)
   {
@@ -6403,13 +6783,13 @@ int main (int argc, char **argv)
   {
     if (show == 1)
     {
-      log_error ("ERROR: Mixing show parameter not supported with keyspace parameter");
+      log_error ("ERROR: Combining show parameter with keyspace parameter is not allowed");
 
       return (-1);
     }
     else if (left == 1)
     {
-      log_error ("ERROR: Mixing left parameter not supported wiht keyspace parameter");
+      log_error ("ERROR: Combining left parameter with keyspace parameter is not allowed");
 
       return (-1);
     }
@@ -6425,6 +6805,26 @@ int main (int argc, char **argv)
     quiet = 1;
   }
 
+  if (stdout_flag == 1)
+  {
+    status_timer          = 0;
+    restore_timer         = 0;
+    restore_disable       = 1;
+    restore               = 0;
+    potfile_disable       = 1;
+    weak_hash_threshold   = 0;
+    gpu_temp_disable      = 1;
+    hash_mode             = 2000;
+    quiet                 = 1;
+    outfile_format        = OUTFILE_FMT_PLAIN;
+    kernel_accel          = 1024;
+    kernel_loops          = 1024;
+    force                 = 1;
+    outfile_check_timer   = 0;
+    session               = "stdout";
+    opencl_vector_width   = 1;
+  }
+
   if (remove_timer_chgd == 1)
   {
     if (remove == 0)
@@ -6517,6 +6917,14 @@ int main (int argc, char **argv)
     weak_hash_threshold = 0;
   }
 
+  if (nvidia_spin_damp > 100)
+  {
+    log_error ("ERROR: setting --nvidia-spin-damp must be between 0 and 100 (inclusive)");
+
+    return (-1);
+  }
+
+
   /**
    * induction directory
    */
@@ -6771,12 +7179,14 @@ int main (int argc, char **argv)
   logfile_top_uint   (attack_mode);
   logfile_top_uint   (attack_kern);
   logfile_top_uint   (benchmark);
+  logfile_top_uint   (stdout_flag);
   logfile_top_uint   (bitmap_min);
   logfile_top_uint   (bitmap_max);
   logfile_top_uint   (debug_mode);
   logfile_top_uint   (force);
   logfile_top_uint   (kernel_accel);
   logfile_top_uint   (kernel_loops);
+  logfile_top_uint   (nvidia_spin_damp);
   logfile_top_uint   (gpu_temp_disable);
   #ifdef HAVE_HWMON
   logfile_top_uint   (gpu_temp_abort);
@@ -6896,15 +7306,21 @@ int main (int argc, char **argv)
     restore_disable       = 1;
     potfile_disable       = 1;
     weak_hash_threshold   = 0;
+    nvidia_spin_damp      = 0;
     gpu_temp_disable      = 1;
+    outfile_check_timer   = 0;
 
     #ifdef HAVE_HWMON
-    powertune_enable      = 1;
+    if (powertune_enable == 1)
+    {
+      gpu_temp_disable = 0;
+    }
     #endif
 
-    data.status_timer     = status_timer;
-    data.restore_timer    = restore_timer;
-    data.restore_disable  = restore_disable;
+    data.status_timer         = status_timer;
+    data.restore_timer        = restore_timer;
+    data.restore_disable      = restore_disable;
+    data.outfile_check_timer  = outfile_check_timer;
 
     /**
      * force attack mode to be bruteforce
@@ -6921,6 +7337,37 @@ int main (int argc, char **argv)
     }
   }
 
+  /**
+   * status, monitor and outfile remove threads
+   */
+
+  uint wordlist_mode = ((optind + 1) < myargc) ? WL_MODE_FILE : WL_MODE_STDIN;
+
+  data.wordlist_mode = wordlist_mode;
+
+  if (wordlist_mode == WL_MODE_STDIN)
+  {
+    status = 1;
+
+    data.status = status;
+  }
+
+  uint outer_threads_cnt = 0;
+
+  hc_thread_t *outer_threads = (hc_thread_t *) mycalloc (10, sizeof (hc_thread_t));
+
+  data.shutdown_outer = 0;
+
+  if (keyspace == 0 && benchmark == 0 && stdout_flag == 0)
+  {
+    if ((data.wordlist_mode == WL_MODE_FILE) || (data.wordlist_mode == WL_MODE_MASK))
+    {
+      hc_thread_create (outer_threads[outer_threads_cnt], thread_keypress, NULL);
+
+      outer_threads_cnt++;
+    }
+  }
+
   /**
    * config
    */
@@ -7675,27 +8122,6 @@ int main (int argc, char **argv)
                    dgst_pos3   = 1;
                    break;
 
-      case   190:  hash_type   = HASH_TYPE_SHA1;
-                   salt_type   = SALT_TYPE_NONE;
-                   attack_exec = ATTACK_EXEC_INSIDE_KERNEL;
-                   opts_type   = OPTS_TYPE_PT_GENERATE_BE
-                               | OPTS_TYPE_PT_ADD80
-                               | OPTS_TYPE_PT_ADDBITS15;
-                   kern_type   = KERN_TYPE_SHA1_LINKEDIN;
-                   dgst_size   = DGST_SIZE_4_5;
-                   parse_func  = sha1linkedin_parse_hash;
-                   sort_by_digest = sort_by_digest_4_5;
-                   opti_type   = OPTI_TYPE_ZERO_BYTE
-                               | OPTI_TYPE_PRECOMPUTE_INIT
-                               | OPTI_TYPE_EARLY_SKIP
-                               | OPTI_TYPE_NOT_ITERATED
-                               | OPTI_TYPE_NOT_SALTED;
-                   dgst_pos0   = 0;
-                   dgst_pos1   = 4;
-                   dgst_pos2   = 3;
-                   dgst_pos3   = 2;
-                   break;
-
       case   200:  hash_type   = HASH_TYPE_MYSQL;
                    salt_type   = SALT_TYPE_NONE;
                    attack_exec = ATTACK_EXEC_INSIDE_KERNEL;
@@ -8336,6 +8762,21 @@ int main (int argc, char **argv)
                    dgst_pos3   = 3;
                    break;
 
+      case  2000:  hash_type   = HASH_TYPE_STDOUT;
+                   salt_type   = SALT_TYPE_NONE;
+                   attack_exec = ATTACK_EXEC_INSIDE_KERNEL;
+                   opts_type   = OPTS_TYPE_PT_GENERATE_LE;
+                   kern_type   = KERN_TYPE_STDOUT;
+                   dgst_size   = DGST_SIZE_4_4;
+                   parse_func  = NULL;
+                   sort_by_digest = NULL;
+                   opti_type   = 0;
+                   dgst_pos0   = 0;
+                   dgst_pos1   = 0;
+                   dgst_pos2   = 0;
+                   dgst_pos3   = 0;
+                   break;
+
       case  2100:  hash_type   = HASH_TYPE_DCC2;
                    salt_type   = SALT_TYPE_EMBEDDED;
                    attack_exec = ATTACK_EXEC_OUTSIDE_KERNEL;
@@ -10750,6 +11191,25 @@ int main (int argc, char **argv)
                    dgst_pos3   = 3;
                    break;
 
+      case 13800:  hash_type   = HASH_TYPE_SHA256;
+                   salt_type   = SALT_TYPE_EMBEDDED;
+                   attack_exec = ATTACK_EXEC_INSIDE_KERNEL;
+                   opts_type   = OPTS_TYPE_PT_GENERATE_BE
+                               | OPTS_TYPE_PT_UNICODE;
+                   kern_type   = KERN_TYPE_WIN8PHONE;
+                   dgst_size   = DGST_SIZE_4_8;
+                   parse_func  = win8phone_parse_hash;
+                   sort_by_digest = sort_by_digest_4_8;
+                   opti_type   = OPTI_TYPE_ZERO_BYTE
+                               | OPTI_TYPE_PRECOMPUTE_INIT
+                               | OPTI_TYPE_EARLY_SKIP
+                               | OPTI_TYPE_NOT_ITERATED
+                               | OPTI_TYPE_RAW_HASH;
+                   dgst_pos0   = 3;
+                   dgst_pos1   = 7;
+                   dgst_pos2   = 2;
+                   dgst_pos3   = 6;
+                   break;
 
       default:     usage_mini_print (PROGNAME); return (-1);
     }
@@ -10876,6 +11336,7 @@ int main (int argc, char **argv)
       case 13761:  esalt_size = sizeof (tc_t);            break;
       case 13762:  esalt_size = sizeof (tc_t);            break;
       case 13763:  esalt_size = sizeof (tc_t);            break;
+      case 13800:  esalt_size = sizeof (win8phone_t);     break;
     }
 
     data.esalt_size = esalt_size;
@@ -11209,7 +11670,7 @@ int main (int argc, char **argv)
                   break;
       case  7400: if (pw_max > 16) pw_max = 16;
                   break;
-      case  7500: if (pw_max >  8) pw_max =  8;
+      case  7700: if (pw_max >  8) pw_max =  8;
                   break;
       case  7900: if (pw_max > 48) pw_max = 48;
                   break;
@@ -11278,7 +11739,7 @@ int main (int argc, char **argv)
 
     uint hashes_avail = 0;
 
-    if (benchmark == 0)
+    if ((benchmark == 0) && (stdout_flag == 0))
     {
       struct stat f;
 
@@ -11460,6 +11921,10 @@ int main (int argc, char **argv)
       {
         // useless to read hash file for keyspace, cheat a little bit w/ optind
       }
+      else if (stdout_flag == 1)
+      {
+        // useless to read hash file for stdout, cheat a little bit w/ optind
+      }
       else if (hashes_avail == 0)
       {
       }
@@ -11483,7 +11948,7 @@ int main (int argc, char **argv)
 
         if (hash_fmt_error)
         {
-          log_info ("WARNING: failed to parse hashes using the '%s' format", strhlfmt (hashlist_format));
+          log_info ("WARNING: Failed to parse hashes using the '%s' format", strhlfmt (hashlist_format));
         }
         else
         {
@@ -11780,7 +12245,7 @@ int main (int argc, char **argv)
 
               if (parser_status < PARSER_GLOBAL_ZERO)
               {
-                log_info ("WARNING: Hashfile '%s' in line %u (%s): %s", data.hashfile, line_num, line_buf, strparser (parser_status));
+                log_info ("WARNING: Hashfile '%s' on line %u (%s): %s", data.hashfile, line_num, line_buf, strparser (parser_status));
 
                 continue;
               }
@@ -11793,7 +12258,7 @@ int main (int argc, char **argv)
 
               if (parser_status < PARSER_GLOBAL_ZERO)
               {
-                log_info ("WARNING: Hashfile '%s' in line %u (%s): %s", data.hashfile, line_num, line_buf, strparser (parser_status));
+                log_info ("WARNING: Hashfile '%s' on line %u (%s): %s", data.hashfile, line_num, line_buf, strparser (parser_status));
 
                 continue;
               }
@@ -11815,7 +12280,7 @@ int main (int argc, char **argv)
 
               if (parser_status < PARSER_GLOBAL_ZERO)
               {
-                log_info ("WARNING: Hashfile '%s' in line %u (%s): %s", data.hashfile, line_num, line_buf, strparser (parser_status));
+                log_info ("WARNING: Hashfile '%s' on line %u (%s): %s", data.hashfile, line_num, line_buf, strparser (parser_status));
 
                 continue;
               }
@@ -11834,7 +12299,7 @@ int main (int argc, char **argv)
 
             if (parser_status < PARSER_GLOBAL_ZERO)
             {
-              log_info ("WARNING: Hashfile '%s' in line %u (%s): %s", data.hashfile, line_num, line_buf, strparser (parser_status));
+              log_info ("WARNING: Hashfile '%s' on line %u (%s): %s", data.hashfile, line_num, line_buf, strparser (parser_status));
 
               continue;
             }
@@ -12269,7 +12734,7 @@ int main (int argc, char **argv)
       return (0);
     }
 
-    if (keyspace == 0)
+    if ((keyspace == 0) && (stdout_flag == 0))
     {
       if (hashes_cnt == 0)
       {
@@ -13058,14 +13523,14 @@ int main (int argc, char **argv)
 
         if (result == -1)
         {
-          log_info ("WARNING: Skipping invalid or unsupported rule in file %s in line %u: %s", rp_file, rule_line, rule_buf);
+          log_info ("WARNING: Skipping invalid or unsupported rule in file %s on line %u: %s", rp_file, rule_line, rule_buf);
 
           continue;
         }
 
         if (cpu_rule_to_kernel_rule (rule_buf, rule_len, &kernel_rules_buf[kernel_rules_cnt]) == -1)
         {
-          log_info ("WARNING: Cannot convert rule for use on device in file %s in line %u: %s", rp_file, rule_line, rule_buf);
+          log_info ("WARNING: Cannot convert rule for use on OpenCL device in file %s on line %u: %s", rp_file, rule_line, rule_buf);
 
           memset (&kernel_rules_buf[kernel_rules_cnt], 0, sizeof (kernel_rule_t)); // needs to be cleared otherwise we could have some remaining data
 
@@ -13075,7 +13540,7 @@ int main (int argc, char **argv)
         /* its so slow
         if (rulefind (&kernel_rules_buf[kernel_rules_cnt], kernel_rules_buf, kernel_rules_cnt, sizeof (kernel_rule_t), sort_by_kernel_rule))
         {
-          log_info ("Duplicate rule for use on device in file %s in line %u: %s", rp_file, rule_line, rule_buf);
+          log_info ("Duplicate rule for use on OpenCL device in file %s in line %u: %s", rp_file, rule_line, rule_buf);
 
           continue;
         }
@@ -13179,7 +13644,7 @@ int main (int argc, char **argv)
      * generate NOP rules
      */
 
-    if (kernel_rules_cnt == 0)
+    if ((rp_files_cnt == 0) && (rp_gen == 0))
     {
       kernel_rules_buf = (kernel_rule_t *) mymalloc (sizeof (kernel_rule_t));
 
@@ -13191,6 +13656,13 @@ int main (int argc, char **argv)
     data.kernel_rules_cnt = kernel_rules_cnt;
     data.kernel_rules_buf = kernel_rules_buf;
 
+    if (kernel_rules_cnt == 0)
+    {
+      log_error ("ERROR: No valid rules left");
+
+      return (-1);
+    }
+
     /**
      * OpenCL platforms: detect
      */
@@ -13232,14 +13704,13 @@ int main (int argc, char **argv)
       }
     }
 
-    /**
-     * OpenCL device types:
-     *   In case the user did not specify --opencl-device-types and the user runs hashcat in a system with only a CPU only he probably want to use that CPU.
-     *   In such a case, automatically enable CPU device type support, since it's disabled by default.
-     */
-
     if (opencl_device_types == NULL)
     {
+      /**
+       * OpenCL device types:
+       *   In case the user did not specify --opencl-device-types and the user runs hashcat in a system with only a CPU only he probably want to use that CPU.
+       */
+
       cl_device_type device_types_all = 0;
 
       for (uint platform_id = 0; platform_id < platforms_cnt; platform_id++)
@@ -13262,19 +13733,33 @@ int main (int argc, char **argv)
         }
       }
 
+      // In such a case, automatically enable CPU device type support, since it's disabled by default.
+
       if ((device_types_all & (CL_DEVICE_TYPE_GPU | CL_DEVICE_TYPE_ACCELERATOR)) == 0)
       {
         device_types_filter |= CL_DEVICE_TYPE_CPU;
       }
+
+      // In another case, when the user uses --stdout, using CPU devices is much faster to setup
+      // If we have a CPU device, force it to be used
+
+      if (stdout_flag == 1)
+      {
+        if (device_types_all & CL_DEVICE_TYPE_CPU)
+        {
+          device_types_filter = CL_DEVICE_TYPE_CPU;
+        }
+      }
     }
 
     /**
      * OpenCL devices: simply push all devices from all platforms into the same device array
      */
 
-    int need_adl   = 0;
-    int need_nvapi = 0;
-    int need_nvml  = 0;
+    int need_adl     = 0;
+    int need_nvapi   = 0;
+    int need_nvml    = 0;
+    int need_xnvctrl = 0;
 
     hc_device_param_t *devices_param = (hc_device_param_t *) mycalloc (DEVICES_MAX, sizeof (hc_device_param_t));
 
@@ -13286,8 +13771,6 @@ int main (int argc, char **argv)
 
     for (uint platform_id = 0; platform_id < platforms_cnt; platform_id++)
     {
-      if ((opencl_platforms_filter & (1 << platform_id)) == 0) continue;
-
       cl_platform_id platform = platforms[platform_id];
 
       hc_clGetDeviceIDs (data.ocl, platform, CL_DEVICE_TYPE_ALL, DEVICES_MAX, platform_devices, &platform_devices_cnt);
@@ -13339,19 +13822,45 @@ int main (int argc, char **argv)
         platform_vendor_id = VENDOR_ID_GENERIC;
       }
 
-      for (uint platform_devices_id = 0; platform_devices_id < platform_devices_cnt; platform_devices_id++)
-      {
-        size_t param_value_size = 0;
-
-        const uint device_id = devices_cnt;
+      const uint platform_skipped = ((opencl_platforms_filter & (1 << platform_id)) == 0);
 
-        hc_device_param_t *device_param = &data.devices_param[device_id];
+      if ((benchmark == 1 || quiet == 0) && (algorithm_pos == 0))
+      {
+        if (machine_readable == 0)
+        {
+          if (platform_skipped == 0)
+          {
+            const int len = log_info ("OpenCL Platform #%u: %s", platform_id + 1, platform_vendor);
 
-        device_param->platform_vendor_id = platform_vendor_id;
+            char line[256] = { 0 };
 
-        device_param->device = platform_devices[platform_devices_id];
+            for (int i = 0; i < len; i++) line[i] = '=';
 
-        device_param->device_id = device_id;
+            log_info (line);
+          }
+          else
+          {
+            log_info ("OpenCL Platform #%u: %s, skipped", platform_id + 1, platform_vendor);
+            log_info ("");
+          }
+        }
+      }
+
+      if (platform_skipped == 1) continue;
+
+      for (uint platform_devices_id = 0; platform_devices_id < platform_devices_cnt; platform_devices_id++)
+      {
+        size_t param_value_size = 0;
+
+        const uint device_id = devices_cnt;
+
+        hc_device_param_t *device_param = &data.devices_param[device_id];
+
+        device_param->platform_vendor_id = platform_vendor_id;
+
+        device_param->device = platform_devices[platform_devices_id];
+
+        device_param->device_id = device_id;
 
         device_param->platform_devices_id = platform_devices_id;
 
@@ -13532,7 +14041,7 @@ int main (int argc, char **argv)
 
         if (device_endian_little == CL_FALSE)
         {
-          log_info ("Device #%u: WARNING: not little endian device", device_id + 1);
+          log_info ("- Device #%u: WARNING: Not a little endian device", device_id + 1);
 
           device_param->skipped = 1;
         }
@@ -13545,7 +14054,7 @@ int main (int argc, char **argv)
 
         if (device_available == CL_FALSE)
         {
-          log_info ("Device #%u: WARNING: device not available", device_id + 1);
+          log_info ("- Device #%u: WARNING: Device not available", device_id + 1);
 
           device_param->skipped = 1;
         }
@@ -13558,7 +14067,7 @@ int main (int argc, char **argv)
 
         if (device_compiler_available == CL_FALSE)
         {
-          log_info ("Device #%u: WARNING: device no compiler available", device_id + 1);
+          log_info ("- Device #%u: WARNING: No compiler available for device", device_id + 1);
 
           device_param->skipped = 1;
         }
@@ -13571,7 +14080,7 @@ int main (int argc, char **argv)
 
         if ((device_execution_capabilities & CL_EXEC_KERNEL) == 0)
         {
-          log_info ("Device #%u: WARNING: device does not support executing kernels", device_id + 1);
+          log_info ("- Device #%u: WARNING: Device does not support executing kernels", device_id + 1);
 
           device_param->skipped = 1;
         }
@@ -13588,14 +14097,14 @@ int main (int argc, char **argv)
 
         if (strstr (device_extensions, "base_atomics") == 0)
         {
-          log_info ("Device #%u: WARNING: device does not support base atomics", device_id + 1);
+          log_info ("- Device #%u: WARNING: Device does not support base atomics", device_id + 1);
 
           device_param->skipped = 1;
         }
 
         if (strstr (device_extensions, "byte_addressable_store") == 0)
         {
-          log_info ("Device #%u: WARNING: device does not support byte addressable store", device_id + 1);
+          log_info ("- Device #%u: WARNING: Device does not support byte addressable store", device_id + 1);
 
           device_param->skipped = 1;
         }
@@ -13610,7 +14119,7 @@ int main (int argc, char **argv)
 
         if (device_local_mem_size < 32768)
         {
-          log_info ("Device #%u: WARNING: device local mem size is too small", device_id + 1);
+          log_info ("- Device #%u: WARNING: Device local mem size is too small", device_id + 1);
 
           device_param->skipped = 1;
         }
@@ -13629,8 +14138,8 @@ int main (int argc, char **argv)
             {
               if (algorithm_pos == 0)
               {
-                log_info ("Device #%u: WARNING: not native intel opencl runtime, expect massive speed loss", device_id + 1);
-                log_info ("           You can use --force to override this but do not post error reports if you do so");
+                log_info ("- Device #%u: WARNING: Not a native Intel OpenCL runtime, expect massive speed loss", device_id + 1);
+                log_info ("             You can use --force to override this but do not post error reports if you do so");
               }
 
               device_param->skipped = 1;
@@ -13671,7 +14180,7 @@ int main (int argc, char **argv)
 
         device_param->device_name_chksum = device_name_chksum;
 
-        // device_processor_cores
+        // vendor specific
 
         if (device_param->device_type & CL_DEVICE_TYPE_GPU)
         {
@@ -13684,34 +14193,19 @@ int main (int argc, char **argv)
           {
             need_nvml = 1;
 
-            #ifdef _WIN
+            #ifdef LINUX
+            need_xnvctrl = 1;
+            #endif
+
+            #ifdef WIN
             need_nvapi = 1;
             #endif
           }
         }
 
-        // device_processor_cores
-
-        if (device_type & CL_DEVICE_TYPE_CPU)
-        {
-          cl_uint device_processor_cores = 1;
-
-          device_param->device_processor_cores = device_processor_cores;
-        }
-
         if (device_type & CL_DEVICE_TYPE_GPU)
         {
-          if (device_vendor_id == VENDOR_ID_AMD)
-          {
-            cl_uint device_processor_cores = 0;
-
-            #define CL_DEVICE_WAVEFRONT_WIDTH_AMD               0x4043
-
-            hc_clGetDeviceInfo (data.ocl, device_param->device, CL_DEVICE_WAVEFRONT_WIDTH_AMD, sizeof (device_processor_cores), &device_processor_cores, NULL);
-
-            device_param->device_processor_cores = device_processor_cores;
-          }
-          else if (device_vendor_id == VENDOR_ID_NV)
+          if (device_vendor_id == VENDOR_ID_NV)
           {
             cl_uint kernel_exec_timeout = 0;
 
@@ -13721,14 +14215,6 @@ int main (int argc, char **argv)
 
             device_param->kernel_exec_timeout = kernel_exec_timeout;
 
-            cl_uint device_processor_cores = 0;
-
-            #define CL_DEVICE_WARP_SIZE_NV                      0x4003
-
-            hc_clGetDeviceInfo (data.ocl, device_param->device, CL_DEVICE_WARP_SIZE_NV, sizeof (device_processor_cores), &device_processor_cores, NULL);
-
-            device_param->device_processor_cores = device_processor_cores;
-
             cl_uint sm_minor = 0;
             cl_uint sm_major = 0;
 
@@ -13740,12 +14226,29 @@ int main (int argc, char **argv)
 
             device_param->sm_minor = sm_minor;
             device_param->sm_major = sm_major;
-          }
-          else
-          {
-            cl_uint device_processor_cores = 1;
 
-            device_param->device_processor_cores = device_processor_cores;
+            // CPU burning loop damper
+            // Value is given as number between 0-100
+            // By default 100%
+
+            device_param->nvidia_spin_damp = (double) nvidia_spin_damp;
+
+            if (nvidia_spin_damp_chgd == 0)
+            {
+              if (data.attack_mode == ATTACK_MODE_STRAIGHT)
+              {
+                /**
+                 * the workaround is not a friend of rule based attacks
+                 * the words from the wordlist combined with fast and slow rules cause
+                 * fluctuations which cause inaccurate wait time estimations
+                 * using a reduced damping percentage almost compensates this
+                 */
+
+                device_param->nvidia_spin_damp = 64;
+              }
+            }
+
+            device_param->nvidia_spin_damp /= 100;
           }
         }
 
@@ -13757,7 +14260,7 @@ int main (int argc, char **argv)
           {
             if (device_param->skipped == 0)
             {
-              log_info ("Device #%u: %s, %lu/%lu MB allocatable, %uMCU",
+              log_info ("- Device #%u: %s, %lu/%lu MB allocatable, %uMCU",
                         device_id + 1,
                         device_name,
                         (unsigned int) (device_maxmem_alloc / 1024 / 1024),
@@ -13766,7 +14269,7 @@ int main (int argc, char **argv)
             }
             else
             {
-              log_info ("Device #%u: %s, skipped",
+              log_info ("- Device #%u: %s, skipped",
                         device_id + 1,
                         device_name);
             }
@@ -13803,8 +14306,8 @@ int main (int argc, char **argv)
               if (catalyst_broken == 1)
               {
                 log_info ("");
-                log_info ("ATTENTION! The installed catalyst driver in your system is known to be broken!");
-                log_info ("It will pass over cracked hashes and does not report them as cracked");
+                log_info ("ATTENTION! The Catalyst driver installed on your system is known to be broken!");
+                log_info ("It passes over cracked hashes and will not report them as cracked");
                 log_info ("You are STRONGLY encouraged not to use it");
                 log_info ("You can use --force to override this but do not post error reports if you do so");
                 log_info ("");
@@ -13815,8 +14318,8 @@ int main (int argc, char **argv)
               if (catalyst_warn == 1)
               {
                 log_info ("");
-                log_info ("ATTENTION! Unsupported or incorrect installed catalyst driver detected!");
-                log_info ("You are STRONGLY encouraged to use the official supported catalyst driver for good reasons");
+                log_info ("ATTENTION! Unsupported or incorrectly installed Catalyst driver detected!");
+                log_info ("You are STRONGLY encouraged to use the official supported catalyst driver");
                 log_info ("See hashcat's homepage for official supported catalyst drivers");
                 #ifdef _WIN
                 log_info ("Also see: http://hashcat.net/wiki/doku.php?id=upgrading_amd_drivers_how_to");
@@ -13831,8 +14334,8 @@ int main (int argc, char **argv)
             {
               if (device_param->kernel_exec_timeout != 0)
               {
-                if (data.quiet == 0) log_info ("Device #%u: WARNING! Kernel exec timeout is not disabled, it might cause you errors of code 702", device_id + 1);
-                if (data.quiet == 0) log_info ("           See the wiki on how to disable it: https://hashcat.net/wiki/doku.php?id=timeout_patch");
+                if (data.quiet == 0) log_info ("- Device #%u: WARNING! Kernel exec timeout is not disabled, it might cause you errors of code 702", device_id + 1);
+                if (data.quiet == 0) log_info ("             See the wiki on how to disable it: https://hashcat.net/wiki/doku.php?id=timeout_patch");
               }
             }
           }
@@ -13921,6 +14424,14 @@ int main (int argc, char **argv)
 
         devices_cnt++;
       }
+
+      if ((benchmark == 1 || quiet == 0) && (algorithm_pos == 0))
+      {
+        if (machine_readable == 0)
+        {
+          log_info ("");
+        }
+      }
     }
 
     if (keyspace == 0 && devices_active == 0)
@@ -13948,32 +14459,27 @@ int main (int argc, char **argv)
 
     data.devices_active = devices_active;
 
-    if ((benchmark == 1 || quiet == 0) && (algorithm_pos == 0))
-    {
-      if (machine_readable == 0)
-      {
-        log_info ("");
-      }
-    }
-
     /**
      * HM devices: init
      */
 
     #ifdef HAVE_HWMON
-    hm_attrs_t hm_adapters_adl[DEVICES_MAX]   = { { 0 } };
-    hm_attrs_t hm_adapters_nvapi[DEVICES_MAX] = { { 0 } };
-    hm_attrs_t hm_adapters_nvml[DEVICES_MAX]  = { { 0 } };
+    hm_attrs_t hm_adapters_adl[DEVICES_MAX]     = { { 0 } };
+    hm_attrs_t hm_adapters_nvapi[DEVICES_MAX]   = { { 0 } };
+    hm_attrs_t hm_adapters_nvml[DEVICES_MAX]    = { { 0 } };
+    hm_attrs_t hm_adapters_xnvctrl[DEVICES_MAX] = { { 0 } };
 
     if (gpu_temp_disable == 0)
     {
-      ADL_PTR   *adl   = (ADL_PTR *)   mymalloc (sizeof (ADL_PTR));
-      NVAPI_PTR *nvapi = (NVAPI_PTR *) mymalloc (sizeof (NVAPI_PTR));
-      NVML_PTR  *nvml  = (NVML_PTR *)  mymalloc (sizeof (NVML_PTR));
+      ADL_PTR     *adl     = (ADL_PTR *)     mymalloc (sizeof (ADL_PTR));
+      NVAPI_PTR   *nvapi   = (NVAPI_PTR *)   mymalloc (sizeof (NVAPI_PTR));
+      NVML_PTR    *nvml    = (NVML_PTR *)    mymalloc (sizeof (NVML_PTR));
+      XNVCTRL_PTR *xnvctrl = (XNVCTRL_PTR *) mymalloc (sizeof (XNVCTRL_PTR));
 
-      data.hm_adl   = NULL;
-      data.hm_nvapi = NULL;
-      data.hm_nvml  = NULL;
+      data.hm_adl     = NULL;
+      data.hm_nvapi   = NULL;
+      data.hm_nvml    = NULL;
+      data.hm_xnvctrl = NULL;
 
       if ((need_nvml == 1) && (nvml_init (nvml) == 0))
       {
@@ -14001,9 +14507,9 @@ int main (int argc, char **argv)
 
             if (hm_NVML_nvmlDeviceGetFanSpeed (data.hm_nvml, 0, hm_adapters_nvml[i].nvml, &speed) == NVML_SUCCESS) hm_adapters_nvml[i].fan_get_supported = 1;
 
-            hm_NVML_nvmlDeviceSetComputeMode (data.hm_nvml, 1, hm_adapters_nvml[i].nvml, NVML_COMPUTEMODE_EXCLUSIVE_PROCESS);
-
-            hm_NVML_nvmlDeviceSetGpuOperationMode (data.hm_nvml, 1, hm_adapters_nvml[i].nvml, NVML_GOM_ALL_ON);
+            // doesn't seem to create any advantages
+            //hm_NVML_nvmlDeviceSetComputeMode (data.hm_nvml, 1, hm_adapters_nvml[i].nvml, NVML_COMPUTEMODE_EXCLUSIVE_PROCESS);
+            //hm_NVML_nvmlDeviceSetGpuOperationMode (data.hm_nvml, 1, hm_adapters_nvml[i].nvml, NVML_GOM_ALL_ON);
           }
         }
       }
@@ -14030,6 +14536,30 @@ int main (int argc, char **argv)
         }
       }
 
+      if ((need_xnvctrl == 1) && (xnvctrl_init (xnvctrl) == 0))
+      {
+        data.hm_xnvctrl = xnvctrl;
+      }
+
+      if (data.hm_xnvctrl)
+      {
+        if (hm_XNVCTRL_XOpenDisplay (data.hm_xnvctrl) == 0)
+        {
+          for (uint device_id = 0; device_id < data.devices_cnt; device_id++)
+          {
+            hc_device_param_t *device_param = &data.devices_param[device_id];
+
+            if ((device_param->device_type & CL_DEVICE_TYPE_GPU) == 0) continue;
+
+            hm_adapters_xnvctrl[device_id].xnvctrl = device_id;
+
+            int speed = 0;
+
+            if (get_fan_speed_current (data.hm_xnvctrl, device_id, &speed) == 0) hm_adapters_xnvctrl[device_id].fan_get_supported = 1;
+          }
+        }
+      }
+
       if ((need_adl == 1) && (adl_init (adl) == 0))
       {
         data.hm_adl = adl;
@@ -14076,7 +14606,7 @@ int main (int argc, char **argv)
         }
       }
 
-      if (data.hm_adl == NULL && data.hm_nvml == NULL)
+      if (data.hm_adl == NULL && data.hm_nvml == NULL && data.hm_xnvctrl == NULL)
       {
         gpu_temp_disable = 1;
       }
@@ -14086,9 +14616,6 @@ int main (int argc, char **argv)
      * OpenCL devices: allocate buffer for device specific information
      */
 
-    int *temp_retain_fanspeed_value  = (int *) mycalloc (data.devices_cnt, sizeof (int));
-    int *temp_retain_fanpolicy_value = (int *) mycalloc (data.devices_cnt, sizeof (int));
-
     ADLOD6MemClockState *od_clock_mem_status = (ADLOD6MemClockState *) mycalloc (data.devices_cnt, sizeof (ADLOD6MemClockState));
 
     int *od_power_control_status = (int *) mycalloc (data.devices_cnt, sizeof (int));
@@ -14109,7 +14636,7 @@ int main (int argc, char **argv)
     {
       if (gpu_temp_abort < gpu_temp_retain)
       {
-        log_error ("ERROR: invalid values for gpu-temp-abort. Parameter gpu-temp-abort is less than gpu-temp-retain.");
+        log_error ("ERROR: Invalid values for gpu-temp-abort. Parameter gpu-temp-abort is less than gpu-temp-retain.");
 
         return (-1);
       }
@@ -14165,7 +14692,7 @@ int main (int argc, char **argv)
        */
 
       #ifdef HAVE_HWMON
-      if (gpu_temp_disable == 0 && data.hm_adl == NULL && data.hm_nvml == NULL)
+      if (gpu_temp_disable == 0 && data.hm_adl == NULL && data.hm_nvml == NULL && data.hm_xnvctrl == NULL)
       {
         log_info ("Watchdog: Hardware Monitoring Interface not found on your system");
       }
@@ -14210,14 +14737,26 @@ int main (int argc, char **argv)
 
         const uint platform_devices_id = device_param->platform_devices_id;
 
-        if (device_param->device_vendor_id == VENDOR_ID_NV)
+        if (device_param->device_vendor_id == VENDOR_ID_AMD)
         {
-          memcpy (&data.hm_device[device_id], &hm_adapters_nvml[platform_devices_id], sizeof (hm_attrs_t));
+          data.hm_device[device_id].adl               = hm_adapters_adl[platform_devices_id].adl;
+          data.hm_device[device_id].nvapi             = 0;
+          data.hm_device[device_id].nvml              = 0;
+          data.hm_device[device_id].xnvctrl           = 0;
+          data.hm_device[device_id].od_version        = hm_adapters_adl[platform_devices_id].od_version;
+          data.hm_device[device_id].fan_get_supported = hm_adapters_adl[platform_devices_id].fan_get_supported;
+          data.hm_device[device_id].fan_set_supported = 0;
         }
 
-        if (device_param->device_vendor_id == VENDOR_ID_AMD)
+        if (device_param->device_vendor_id == VENDOR_ID_NV)
         {
-          memcpy (&data.hm_device[device_id], &hm_adapters_adl[platform_devices_id], sizeof (hm_attrs_t));
+          data.hm_device[device_id].adl               = 0;
+          data.hm_device[device_id].nvapi             = hm_adapters_nvapi[platform_devices_id].nvapi;
+          data.hm_device[device_id].nvml              = hm_adapters_nvml[platform_devices_id].nvml;
+          data.hm_device[device_id].xnvctrl           = hm_adapters_xnvctrl[platform_devices_id].xnvctrl;
+          data.hm_device[device_id].od_version        = 0;
+          data.hm_device[device_id].fan_get_supported = hm_adapters_nvml[platform_devices_id].fan_get_supported;
+          data.hm_device[device_id].fan_set_supported = 0;
         }
       }
     }
@@ -14325,12 +14864,12 @@ int main (int argc, char **argv)
 
               if ((engine_clock_max - engine_clock_profile_max) > warning_trigger_engine)
               {
-                log_info ("WARN: the custom profile seems to have too low maximum engine clock values. You therefore may not reach full performance");
+                log_info ("WARN: The custom profile seems to have too low maximum engine clock values. You therefore may not reach full performance");
               }
 
               if ((memory_clock_max - memory_clock_profile_max) > warning_trigger_memory)
               {
-                log_info ("WARN: the custom profile seems to have too low maximum memory clock values. You therefore may not reach full performance");
+                log_info ("WARN: The custom profile seems to have too low maximum memory clock values. You therefore may not reach full performance");
               }
 
               ADLOD6StateInfo *performance_state = (ADLOD6StateInfo*) mycalloc (1, sizeof (ADLOD6StateInfo) + sizeof (ADLOD6PerformanceLevel));
@@ -14439,7 +14978,6 @@ int main (int argc, char **argv)
 
       const char *device_name_chksum      = device_param->device_name_chksum;
       const u32   device_processors       = device_param->device_processors;
-      const u32   device_processor_cores  = device_param->device_processor_cores;
 
       /**
        * create context for each device
@@ -14473,7 +15011,9 @@ int main (int argc, char **argv)
       if (hash_mode ==  3000) kernel_threads = 64; // DES
       if (hash_mode ==  3200) kernel_threads = 8;  // Blowfish
       if (hash_mode ==  7500) kernel_threads = 64; // RC4
+      if (hash_mode ==  8900) kernel_threads = 64; // Scrypt
       if (hash_mode ==  9000) kernel_threads = 8;  // Blowfish
+      if (hash_mode ==  9300) kernel_threads = 64; // Scrypt
       if (hash_mode ==  9700) kernel_threads = 64; // RC4
       if (hash_mode ==  9710) kernel_threads = 64; // RC4
       if (hash_mode ==  9800) kernel_threads = 64; // RC4
@@ -14485,6 +15025,8 @@ int main (int argc, char **argv)
 
       device_param->kernel_threads = kernel_threads;
 
+      device_param->hardware_power = device_processors * kernel_threads;
+
       /**
        * create input buffers on device : calculate size of fixed memory buffers
        */
@@ -14517,10 +15059,28 @@ int main (int argc, char **argv)
 
       // scryptV stuff
 
-      size_t size_scryptV = 1;
+      size_t size_scrypt = 4;
 
       if ((hash_mode == 8900) || (hash_mode == 9300))
       {
+        // we need to check that all hashes have the same scrypt settings
+
+        const u32 scrypt_N = data.salts_buf[0].scrypt_N;
+        const u32 scrypt_r = data.salts_buf[0].scrypt_r;
+        const u32 scrypt_p = data.salts_buf[0].scrypt_p;
+
+        for (uint i = 1; i < salts_cnt; i++)
+        {
+          if ((data.salts_buf[i].scrypt_N != scrypt_N)
+           || (data.salts_buf[i].scrypt_r != scrypt_r)
+           || (data.salts_buf[i].scrypt_p != scrypt_p))
+          {
+            log_error ("ERROR: Mixed scrypt settings not supported");
+
+            return -1;
+          }
+        }
+
         uint tmto_start = 0;
         uint tmto_stop  = 10;
 
@@ -14531,14 +15091,13 @@ int main (int argc, char **argv)
         else
         {
           // in case the user did not specify the tmto manually
-          // use some values known to run best (tested on 290x for AMD and 980ti for NV)
-          // but set the lower end only in case the user has a device with too less memory
+          // use some values known to run best (tested on 290x for AMD and GTX1080 for NV)
 
           if (hash_mode == 8900)
           {
             if (device_param->device_vendor_id == VENDOR_ID_AMD)
             {
-              tmto_start = 1;
+              tmto_start = 3;
             }
             else if (device_param->device_vendor_id == VENDOR_ID_NV)
             {
@@ -14553,53 +15112,65 @@ int main (int argc, char **argv)
             }
             else if (device_param->device_vendor_id == VENDOR_ID_NV)
             {
-              tmto_start = 2;
+              tmto_start = 4;
             }
           }
         }
 
-        for (uint tmto = tmto_start; tmto < tmto_stop; tmto++)
+        data.scrypt_tmp_size = (128 * scrypt_r);
+
+        device_param->kernel_accel_min = 1;
+        device_param->kernel_accel_max = 8;
+
+        uint tmto;
+
+        for (tmto = tmto_start; tmto < tmto_stop; tmto++)
         {
-          // TODO: in theory the following calculation needs to be done per salt, not global
-          //       we assume all hashes have the same scrypt settings
+          size_scrypt = (128 * scrypt_r) * scrypt_N;
+
+          size_scrypt /= 1 << tmto;
 
-          size_scryptV = (128 * data.salts_buf[0].scrypt_r) * data.salts_buf[0].scrypt_N;
+          size_scrypt *= device_param->device_processors * device_param->kernel_threads * device_param->kernel_accel_max;
 
-          size_scryptV /= 1 << tmto;
+          if ((size_scrypt / 4) > device_param->device_maxmem_alloc)
+          {
+            if (quiet == 0) log_info ("WARNING: Not enough single-block device memory allocatable to use --scrypt-tmto %d, increasing...", tmto);
 
-          size_scryptV *= device_processors * device_processor_cores;
+            continue;
+          }
 
-          if (size_scryptV > device_param->device_maxmem_alloc)
+          if (size_scrypt > device_param->device_global_mem)
           {
-            if (quiet == 0) log_info ("WARNING: not enough device memory allocatable to use --scrypt-tmto %d, increasing...", tmto);
+            if (quiet == 0) log_info ("WARNING: Not enough total device memory allocatable to use --scrypt-tmto %d, increasing...", tmto);
 
             continue;
           }
 
           for (uint salts_pos = 0; salts_pos < data.salts_cnt; salts_pos++)
           {
-            data.salts_buf[salts_pos].scrypt_tmto = tmto;
-            data.salts_buf[salts_pos].scrypt_phy  = device_processors * device_processor_cores;
+            data.scrypt_tmto_final = tmto;
           }
 
           break;
         }
 
-        if (data.salts_buf[0].scrypt_phy == 0)
+        if (tmto == tmto_stop)
         {
-          log_error ("ERROR: can't allocate enough device memory");
+          log_error ("ERROR: Can't allocate enough device memory");
 
           return -1;
         }
 
-        if (quiet == 0) log_info ("SCRYPT tmto optimizer value set to: %u, mem: %u\n", data.salts_buf[0].scrypt_tmto, size_scryptV);
+        if (quiet == 0) log_info ("SCRYPT tmto optimizer value set to: %u, mem: %u\n", data.scrypt_tmto_final, size_scrypt);
       }
 
+      size_t size_scrypt4 = size_scrypt / 4;
+
       /**
        * some algorithms need a fixed kernel-loops count
        */
 
-      if (hash_mode == 1500)
+      if (hash_mode == 1500 && attack_mode == ATTACK_MODE_BF)
       {
         const u32 kernel_loops_fixed = 1024;
 
@@ -14607,7 +15178,7 @@ int main (int argc, char **argv)
         device_param->kernel_loops_max = kernel_loops_fixed;
       }
 
-      if (hash_mode == 3000)
+      if (hash_mode == 3000 && attack_mode == ATTACK_MODE_BF)
       {
         const u32 kernel_loops_fixed = 1024;
 
@@ -14720,11 +15291,11 @@ int main (int argc, char **argv)
           case  7900: size_tmps = kernel_power_max * sizeof (drupal7_tmp_t);         break;
           case  8200: size_tmps = kernel_power_max * sizeof (pbkdf2_sha512_tmp_t);   break;
           case  8800: size_tmps = kernel_power_max * sizeof (androidfde_tmp_t);      break;
-          case  8900: size_tmps = kernel_power_max * sizeof (scrypt_tmp_t);          break;
+          case  8900: size_tmps = kernel_power_max * data.scrypt_tmp_size;           break;
           case  9000: size_tmps = kernel_power_max * sizeof (pwsafe2_tmp_t);         break;
           case  9100: size_tmps = kernel_power_max * sizeof (lotus8_tmp_t);          break;
           case  9200: size_tmps = kernel_power_max * sizeof (pbkdf2_sha256_tmp_t);   break;
-          case  9300: size_tmps = kernel_power_max * sizeof (scrypt_tmp_t);          break;
+          case  9300: size_tmps = kernel_power_max * data.scrypt_tmp_size;           break;
           case  9400: size_tmps = kernel_power_max * sizeof (office2007_tmp_t);      break;
           case  9500: size_tmps = kernel_power_max * sizeof (office2010_tmp_t);      break;
           case  9600: size_tmps = kernel_power_max * sizeof (office2013_tmp_t);      break;
@@ -14774,13 +15345,19 @@ int main (int argc, char **argv)
 
         if ((opts_type & OPTS_TYPE_HOOK12) || (opts_type & OPTS_TYPE_HOOK23))
         {
-          // none yet
+          switch (hash_mode)
+          {
+          }
         }
 
         // now check if all device-memory sizes which depend on the kernel_accel_max amplifier are within its boundaries
         // if not, decrease amplifier and try again
 
-        int skip = 0;
+        int memory_limit_hit = 0;
+
+        if (size_pws   > device_param->device_maxmem_alloc) memory_limit_hit = 1;
+        if (size_tmps  > device_param->device_maxmem_alloc) memory_limit_hit = 1;
+        if (size_hooks > device_param->device_maxmem_alloc) memory_limit_hit = 1;
 
         const u64 size_total
           = bitmap_size
@@ -14805,17 +15382,17 @@ int main (int argc, char **argv)
           + size_rules
           + size_rules_c
           + size_salts
-          + size_scryptV
+          + size_scrypt4
+          + size_scrypt4
+          + size_scrypt4
+          + size_scrypt4
           + size_shown
           + size_tm
           + size_tmps;
 
-        // Don't ask me, ask AMD!
-
-        if (size_total > device_param->device_maxmem_alloc) skip = 1;
-        if (size_total > device_param->device_global_mem)   skip = 1;
+        if (size_total > device_param->device_global_mem) memory_limit_hit = 1;
 
-        if (skip == 1)
+        if (memory_limit_hit == 1)
         {
           kernel_accel_max--;
 
@@ -14825,14 +15402,12 @@ int main (int argc, char **argv)
         break;
       }
 
-      /*
-      if (kernel_accel_max == 0)
+      if (kernel_accel_max < kernel_accel_min)
       {
-        log_error ("Device #%u: Device does not provide enough allocatable device-memory to handle hash-type %u", device_id + 1, data.hash_mode);
+        log_error ("- Device #%u: Device does not provide enough allocatable device-memory to handle this attack", device_id + 1);
 
         return -1;
       }
-      */
 
       device_param->kernel_accel_min = kernel_accel_min;
       device_param->kernel_accel_max = kernel_accel_max;
@@ -14840,7 +15415,7 @@ int main (int argc, char **argv)
       /*
       if (kernel_accel_max < kernel_accel)
       {
-        if (quiet == 0) log_info ("Device #%u: Reduced maximum kernel-accel to %u", device_id + 1, kernel_accel_max);
+        if (quiet == 0) log_info ("- Device #%u: Reduced maximum kernel-accel to %u", device_id + 1, kernel_accel_max);
 
         device_param->kernel_accel = kernel_accel_max;
       }
@@ -14858,35 +15433,114 @@ int main (int argc, char **argv)
        * default building options
        */
 
-      char build_opts[1024] = { 0 };
+      char cpath[1024] = { 0 };
 
-      // we don't have sm_* on vendors not NV but it doesn't matter
+      char build_opts[1024] = { 0 };
 
       #if _WIN
-      snprintf (build_opts, sizeof (build_opts) - 1, "-I \"%s\\OpenCL\\\" -I '%s\\OpenCL\\' -I %s\\OpenCL\\ -I\"%s\\OpenCL\\\" -I'%s\\OpenCL\\' -I%s\\OpenCL\\", shared_dir, shared_dir, shared_dir, shared_dir, shared_dir, shared_dir);
+
+      snprintf (cpath, sizeof (cpath) - 1, "%s\\OpenCL\\", shared_dir);
+
+      char *cpath_real = mymalloc (MAX_PATH);
+
+      if (GetFullPathName (cpath, MAX_PATH, cpath_real, NULL) == 0)
+      {
+        log_error ("ERROR: %s: %s", cpath, "GetFullPathName()");
+
+        return -1;
+      }
+
+      naive_replace (cpath_real, '\\', '/');
+
+      // not escaping here, windows has quotes
+
+      snprintf (build_opts, sizeof (build_opts) - 1, "-I \"%s\"", cpath_real);
+
       #else
-      snprintf (build_opts, sizeof (build_opts) - 1, "-I \"%s/OpenCL/\" -I '%s/OpenCL/' -I %s/OpenCL/ -I\"%s/OpenCL/\" -I'%s/OpenCL/' -I%s/OpenCL/", shared_dir, shared_dir, shared_dir, shared_dir, shared_dir, shared_dir);
+
+      snprintf (cpath, sizeof (cpath) - 1, "%s/OpenCL/", shared_dir);
+
+      char *cpath_real = mymalloc (PATH_MAX);
+
+      if (realpath (cpath, cpath_real) == NULL)
+      {
+        log_error ("ERROR: %s: %s", cpath, strerror (errno));
+
+        return -1;
+      }
+
+      naive_escape (cpath_real, PATH_MAX,  ' ', '\\');
+
+      snprintf (build_opts, sizeof (build_opts) - 1, "-I %s", cpath_real);
+
       #endif
 
-      char build_opts_new[1024] = { 0 };
+      // include check
+      // this test needs to be done manually because of osx opencl runtime
+      // if there's a problem with permission, its not reporting back and erroring out silently
 
-      snprintf (build_opts_new, sizeof (build_opts_new) - 1, "%s -DVENDOR_ID=%u -DCUDA_ARCH=%d -DVECT_SIZE=%u -DDEVICE_TYPE=%u -DKERN_TYPE=%u -D_unroll -cl-std=CL1.1", build_opts, device_param->device_vendor_id, (device_param->sm_major * 100) + device_param->sm_minor, device_param->vector_width, (u32) device_param->device_type, kern_type);
+      #define files_cnt 15
 
-      strncpy (build_opts, build_opts_new, sizeof (build_opts) - 1);
+      const char *files_names[files_cnt] =
+      {
+        "inc_cipher_aes256.cl",
+        "inc_cipher_serpent256.cl",
+        "inc_cipher_twofish256.cl",
+        "inc_common.cl",
+        "inc_comp_multi_bs.cl",
+        "inc_comp_multi.cl",
+        "inc_comp_single_bs.cl",
+        "inc_comp_single.cl",
+        "inc_hash_constants.h",
+        "inc_hash_functions.cl",
+        "inc_rp.cl",
+        "inc_rp.h",
+        "inc_simd.cl",
+        "inc_types.cl",
+        "inc_vendor.cl",
+      };
 
-      /*
-      if (device_param->device_vendor_id == VENDOR_ID_INTEL_SDK)
+      for (int i = 0; i < files_cnt; i++)
       {
-        // we do vectorizing much better than the auto-vectorizer
+        char path[1024] = { 0 };
+
+        snprintf (path, sizeof (path) - 1, "%s/%s", cpath_real, files_names[i]);
+
+        FILE *fd = fopen (path, "r");
+
+        if (fd == NULL)
+        {
+          log_error ("ERROR: %s: fopen(): %s", path, strerror (errno));
+
+          return -1;
+        }
 
-        snprintf (build_opts_new, sizeof (build_opts_new) - 1, "%s -cl-opt-disable", build_opts);
+        char buf[1];
 
-        strncpy (build_opts, build_opts_new, sizeof (build_opts) - 1);
+        size_t n = fread (buf, 1, 1, fd);
+
+        if (n != 1)
+        {
+          log_error ("ERROR: %s: fread(): %s", path, strerror (errno));
+
+          return -1;
+        }
+
+        fclose (fd);
       }
-      */
+
+      myfree (cpath_real);
+
+      // we don't have sm_* on vendors not NV but it doesn't matter
+
+      char build_opts_new[1024] = { 0 };
+
+      snprintf (build_opts_new, sizeof (build_opts_new) - 1, "%s -D VENDOR_ID=%u -D CUDA_ARCH=%d -D VECT_SIZE=%u -D DEVICE_TYPE=%u -D DGST_R0=%u -D DGST_R1=%u -D DGST_R2=%u -D DGST_R3=%u -D DGST_ELEM=%u -D KERN_TYPE=%u -D _unroll -cl-std=CL1.1", build_opts, device_param->device_vendor_id, (device_param->sm_major * 100) + device_param->sm_minor, device_param->vector_width, (u32) device_param->device_type, data.dgst_pos0, data.dgst_pos1, data.dgst_pos2, data.dgst_pos3, data.dgst_size / 4, kern_type);
+
+      strncpy (build_opts, build_opts_new, sizeof (build_opts));
 
       #ifdef DEBUG
-      log_info ("Device #%u: build_opts '%s'\n", device_id + 1, build_opts);
+      log_info ("- Device #%u: build_opts '%s'\n", device_id + 1, build_opts);
       #endif
 
       /**
@@ -14940,7 +15594,7 @@ int main (int argc, char **argv)
         {
           if (cached == 0)
           {
-            if (quiet == 0) log_info ("Device #%u: Kernel %s not found in cache! Building may take a while...", device_id + 1, cached_file);
+            if (quiet == 0) log_info ("- Device #%u: Kernel %s not found in cache! Building may take a while...", device_id + 1, filename_from_filepath (cached_file));
 
             load_kernel (source_file, 1, kernel_lengths, kernel_sources);
 
@@ -14971,7 +15625,7 @@ int main (int argc, char **argv)
             {
               device_param->skipped = true;
 
-              log_info ("Device #%u: Kernel %s build failure. Proceed without this device.", device_id + 1, source_file);
+              log_info ("- Device #%u: Kernel %s build failure. Proceeding without this device.", device_id + 1, source_file);
 
               continue;
             }
@@ -14991,7 +15645,7 @@ int main (int argc, char **argv)
           else
           {
             #ifdef DEBUG
-            log_info ("Device #%u: Kernel %s (%ld bytes)", device_id + 1, cached_file, cst.st_size);
+            log_info ("- Device #%u: Kernel %s (%ld bytes)", device_id + 1, cached_file, cst.st_size);
             #endif
 
             load_kernel (cached_file, 1, kernel_lengths, kernel_sources);
@@ -15004,7 +15658,7 @@ int main (int argc, char **argv)
         else
         {
           #ifdef DEBUG
-          log_info ("Device #%u: Kernel %s (%ld bytes)", device_id + 1, source_file, sst.st_size);
+          log_info ("- Device #%u: Kernel %s (%ld bytes)", device_id + 1, source_file, sst.st_size);
           #endif
 
           load_kernel (source_file, 1, kernel_lengths, kernel_sources);
@@ -15015,11 +15669,11 @@ int main (int argc, char **argv)
 
           if (force_jit_compilation == 1500)
           {
-            snprintf (build_opts_update, sizeof (build_opts_update) - 1, "%s -DDESCRYPT_SALT=%d", build_opts, data.salts_buf[0].salt_buf[0]);
+            snprintf (build_opts_update, sizeof (build_opts_update) - 1, "%s -DDESCRYPT_SALT=%u", build_opts, data.salts_buf[0].salt_buf[0]);
           }
           else if (force_jit_compilation == 8900)
           {
-            snprintf (build_opts_update, sizeof (build_opts_update) - 1, "%s -DSCRYPT_N=%d -DSCRYPT_R=%d -DSCRYPT_P=%d -DSCRYPT_TMTO=%d", build_opts, data.salts_buf[0].scrypt_N, data.salts_buf[0].scrypt_r, data.salts_buf[0].scrypt_p, 1 << data.salts_buf[0].scrypt_tmto);
+            snprintf (build_opts_update, sizeof (build_opts_update) - 1, "%s -DSCRYPT_N=%u -DSCRYPT_R=%u -DSCRYPT_P=%u -DSCRYPT_TMTO=%u -DSCRYPT_TMP_ELEM=%u", build_opts, data.salts_buf[0].scrypt_N, data.salts_buf[0].scrypt_r, data.salts_buf[0].scrypt_p, 1 << data.scrypt_tmto_final, data.scrypt_tmp_size / 16);
           }
           else
           {
@@ -15051,7 +15705,7 @@ int main (int argc, char **argv)
           {
             device_param->skipped = true;
 
-            log_info ("Device #%u: Kernel %s build failure. Proceed without this device.", device_id + 1, source_file);
+            log_info ("- Device #%u: Kernel %s build failure. Proceeding without this device.", device_id + 1, source_file);
           }
         }
 
@@ -15110,7 +15764,7 @@ int main (int argc, char **argv)
 
         if (cached == 0)
         {
-          if (quiet == 0) log_info ("Device #%u: Kernel %s not found in cache! Building may take a while...", device_id + 1, cached_file);
+          if (quiet == 0) log_info ("- Device #%u: Kernel %s not found in cache! Building may take a while...", device_id + 1, filename_from_filepath (cached_file));
           if (quiet == 0) log_info ("");
 
           load_kernel (source_file, 1, kernel_lengths, kernel_sources);
@@ -15123,7 +15777,7 @@ int main (int argc, char **argv)
           {
             device_param->skipped = true;
 
-            log_info ("Device #%u: Kernel %s build failure. Proceed without this device.", device_id + 1, source_file);
+            log_info ("- Device #%u: Kernel %s build failure. Proceeding without this device.", device_id + 1, source_file);
 
             continue;
           }
@@ -15143,7 +15797,7 @@ int main (int argc, char **argv)
         else
         {
           #ifdef DEBUG
-          log_info ("Device #%u: Kernel %s (%ld bytes)", device_id + 1, cached_file, cst.st_size);
+          log_info ("- Device #%u: Kernel %s (%ld bytes)", device_id + 1, cached_file, cst.st_size);
           #endif
 
           load_kernel (cached_file, 1, kernel_lengths, kernel_sources);
@@ -15212,7 +15866,7 @@ int main (int argc, char **argv)
 
         if (cached == 0)
         {
-          if (quiet == 0) log_info ("Device #%u: Kernel %s not found in cache! Building may take a while...", device_id + 1, cached_file);
+          if (quiet == 0) log_info ("- Device #%u: Kernel %s not found in cache! Building may take a while...", device_id + 1, filename_from_filepath (cached_file));
           if (quiet == 0) log_info ("");
 
           load_kernel (source_file, 1, kernel_lengths, kernel_sources);
@@ -15225,7 +15879,7 @@ int main (int argc, char **argv)
           {
             device_param->skipped = true;
 
-            log_info ("Device #%u: Kernel %s build failure. Proceed without this device.", device_id + 1, source_file);
+            log_info ("- Device #%u: Kernel %s build failure. Proceed without this device.", device_id + 1, source_file);
 
             continue;
           }
@@ -15245,7 +15899,7 @@ int main (int argc, char **argv)
         else
         {
           #ifdef DEBUG
-          if (quiet == 0) log_info ("Device #%u: Kernel %s (%ld bytes)", device_id + 1, cached_file, cst.st_size);
+          if (quiet == 0) log_info ("- Device #%u: Kernel %s (%ld bytes)", device_id + 1, cached_file, cst.st_size);
           #endif
 
           load_kernel (cached_file, 1, kernel_lengths, kernel_sources);
@@ -15291,7 +15945,10 @@ int main (int argc, char **argv)
       device_param->d_digests_shown = hc_clCreateBuffer (data.ocl, device_param->context, CL_MEM_READ_WRITE,  size_shown,   NULL);
       device_param->d_salt_bufs     = hc_clCreateBuffer (data.ocl, device_param->context, CL_MEM_READ_ONLY,   size_salts,   NULL);
       device_param->d_result        = hc_clCreateBuffer (data.ocl, device_param->context, CL_MEM_READ_WRITE,  size_results, NULL);
-      device_param->d_scryptV_buf   = hc_clCreateBuffer (data.ocl, device_param->context, CL_MEM_READ_WRITE,  size_scryptV, NULL);
+      device_param->d_scryptV0_buf  = hc_clCreateBuffer (data.ocl, device_param->context, CL_MEM_READ_WRITE,  size_scrypt4, NULL);
+      device_param->d_scryptV1_buf  = hc_clCreateBuffer (data.ocl, device_param->context, CL_MEM_READ_WRITE,  size_scrypt4, NULL);
+      device_param->d_scryptV2_buf  = hc_clCreateBuffer (data.ocl, device_param->context, CL_MEM_READ_WRITE,  size_scrypt4, NULL);
+      device_param->d_scryptV3_buf  = hc_clCreateBuffer (data.ocl, device_param->context, CL_MEM_READ_WRITE,  size_scrypt4, NULL);
 
       hc_clEnqueueWriteBuffer (data.ocl, device_param->command_queue, device_param->d_bitmap_s1_a,    CL_TRUE, 0, bitmap_size,  bitmap_s1_a,        0, NULL, NULL);
       hc_clEnqueueWriteBuffer (data.ocl, device_param->command_queue, device_param->d_bitmap_s1_b,    CL_TRUE, 0, bitmap_size,  bitmap_s1_b,        0, NULL, NULL);
@@ -15359,17 +16016,17 @@ int main (int argc, char **argv)
        * kernel args
        */
 
-      device_param->kernel_params_buf32[21] = bitmap_mask;
-      device_param->kernel_params_buf32[22] = bitmap_shift1;
-      device_param->kernel_params_buf32[23] = bitmap_shift2;
-      device_param->kernel_params_buf32[24] = 0; // salt_pos
-      device_param->kernel_params_buf32[25] = 0; // loop_pos
-      device_param->kernel_params_buf32[26] = 0; // loop_cnt
-      device_param->kernel_params_buf32[27] = 0; // kernel_rules_cnt
-      device_param->kernel_params_buf32[28] = 0; // digests_cnt
-      device_param->kernel_params_buf32[29] = 0; // digests_offset
-      device_param->kernel_params_buf32[30] = 0; // combs_mode
-      device_param->kernel_params_buf32[31] = 0; // gid_max
+      device_param->kernel_params_buf32[24] = bitmap_mask;
+      device_param->kernel_params_buf32[25] = bitmap_shift1;
+      device_param->kernel_params_buf32[26] = bitmap_shift2;
+      device_param->kernel_params_buf32[27] = 0; // salt_pos
+      device_param->kernel_params_buf32[28] = 0; // loop_pos
+      device_param->kernel_params_buf32[29] = 0; // loop_cnt
+      device_param->kernel_params_buf32[30] = 0; // kernel_rules_cnt
+      device_param->kernel_params_buf32[31] = 0; // digests_cnt
+      device_param->kernel_params_buf32[32] = 0; // digests_offset
+      device_param->kernel_params_buf32[33] = 0; // combs_mode
+      device_param->kernel_params_buf32[34] = 0; // gid_max
 
       device_param->kernel_params[ 0] = (attack_exec == ATTACK_EXEC_INSIDE_KERNEL)
                                       ? &device_param->d_pws_buf
@@ -15393,10 +16050,10 @@ int main (int argc, char **argv)
       device_param->kernel_params[17] = &device_param->d_salt_bufs;
       device_param->kernel_params[18] = &device_param->d_esalt_bufs;
       device_param->kernel_params[19] = &device_param->d_result;
-      device_param->kernel_params[20] = &device_param->d_scryptV_buf;
-      device_param->kernel_params[21] = &device_param->kernel_params_buf32[21];
-      device_param->kernel_params[22] = &device_param->kernel_params_buf32[22];
-      device_param->kernel_params[23] = &device_param->kernel_params_buf32[23];
+      device_param->kernel_params[20] = &device_param->d_scryptV0_buf;
+      device_param->kernel_params[21] = &device_param->d_scryptV1_buf;
+      device_param->kernel_params[22] = &device_param->d_scryptV2_buf;
+      device_param->kernel_params[23] = &device_param->d_scryptV3_buf;
       device_param->kernel_params[24] = &device_param->kernel_params_buf32[24];
       device_param->kernel_params[25] = &device_param->kernel_params_buf32[25];
       device_param->kernel_params[26] = &device_param->kernel_params_buf32[26];
@@ -15405,6 +16062,9 @@ int main (int argc, char **argv)
       device_param->kernel_params[29] = &device_param->kernel_params_buf32[29];
       device_param->kernel_params[30] = &device_param->kernel_params_buf32[30];
       device_param->kernel_params[31] = &device_param->kernel_params_buf32[31];
+      device_param->kernel_params[32] = &device_param->kernel_params_buf32[32];
+      device_param->kernel_params[33] = &device_param->kernel_params_buf32[33];
+      device_param->kernel_params[34] = &device_param->kernel_params_buf32[34];
 
       device_param->kernel_params_mp_buf64[3] = 0;
       device_param->kernel_params_mp_buf32[4] = 0;
@@ -15568,7 +16228,7 @@ int main (int argc, char **argv)
       hc_clGetKernelWorkGroupInfo (data.ocl, device_param->kernel2, device_param->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof (size_t), &kernel_wgs_tmp, NULL); kernel_threads = MIN (kernel_threads, kernel_wgs_tmp);
       hc_clGetKernelWorkGroupInfo (data.ocl, device_param->kernel3, device_param->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof (size_t), &kernel_wgs_tmp, NULL); kernel_threads = MIN (kernel_threads, kernel_wgs_tmp);
 
-      for (uint i = 0; i <= 20; i++)
+      for (uint i = 0; i <= 23; i++)
       {
         hc_clSetKernelArg (data.ocl, device_param->kernel1, i, sizeof (cl_mem), device_param->kernel_params[i]);
         hc_clSetKernelArg (data.ocl, device_param->kernel2, i, sizeof (cl_mem), device_param->kernel_params[i]);
@@ -15578,7 +16238,7 @@ int main (int argc, char **argv)
         if (opts_type & OPTS_TYPE_HOOK23) hc_clSetKernelArg (data.ocl, device_param->kernel23, i, sizeof (cl_mem), device_param->kernel_params[i]);
       }
 
-      for (uint i = 21; i <= 31; i++)
+      for (uint i = 24; i <= 34; i++)
       {
         hc_clSetKernelArg (data.ocl, device_param->kernel1, i, sizeof (cl_uint), device_param->kernel_params[i]);
         hc_clSetKernelArg (data.ocl, device_param->kernel2, i, sizeof (cl_uint), device_param->kernel_params[i]);
@@ -15710,9 +16370,6 @@ int main (int argc, char **argv)
             const int fanspeed  = hm_get_fanspeed_with_device_id  (device_id);
             const int fanpolicy = hm_get_fanpolicy_with_device_id (device_id);
 
-            temp_retain_fanspeed_value[device_id]  = fanspeed;
-            temp_retain_fanpolicy_value[device_id] = fanpolicy;
-
             // we also set it to tell the OS we take control over the fan and it's automatic controller
             // if it was set to automatic. we do not control user-defined fanspeeds.
 
@@ -15728,7 +16385,13 @@ int main (int argc, char **argv)
               }
               else if (device_param->device_vendor_id == VENDOR_ID_NV)
               {
+                #ifdef LINUX
+                rc = set_fan_control (data.hm_xnvctrl, data.hm_device[device_id].xnvctrl, NV_CTRL_GPU_COOLER_MANUAL_CONTROL_TRUE);
+                #endif
 
+                #ifdef WIN
+                rc = hm_set_fanspeed_with_device_id_nvapi (device_id, fanspeed, 1);
+                #endif
               }
 
               if (rc == 0)
@@ -15827,10 +16490,6 @@ int main (int argc, char **argv)
     wl_data->cnt   = 0;
     wl_data->pos   = 0;
 
-    uint wordlist_mode = ((optind + 1) < myargc) ? WL_MODE_FILE : WL_MODE_STDIN;
-
-    data.wordlist_mode = wordlist_mode;
-
     cs_t  *css_buf   = NULL;
     uint   css_cnt   = 0;
     uint   dictcnt   = 0;
@@ -15875,7 +16534,7 @@ int main (int argc, char **argv)
 
             if (keyspace == 1)
             {
-              log_error ("ERROR: keyspace parameter is not allowed together with a directory");
+              log_error ("ERROR: Keyspace parameter is not allowed together with a directory");
 
               return (-1);
             }
@@ -16338,7 +16997,7 @@ int main (int argc, char **argv)
 
           if (keyspace == 1)
           {
-            log_error ("ERROR: keyspace parameter is not allowed together with a directory");
+            log_error ("ERROR: Keyspace parameter is not allowed together with a directory");
 
             return (-1);
           }
@@ -16517,7 +17176,7 @@ int main (int argc, char **argv)
 
           if (keyspace == 1)
           {
-            log_error ("ERROR: keyspace parameter is not allowed together with a directory");
+            log_error ("ERROR: Keyspace parameter is not allowed together with a directory");
 
             return (-1);
           }
@@ -16623,31 +17282,27 @@ int main (int argc, char **argv)
      * status and monitor threads
      */
 
-    if (data.devices_status != STATUS_CRACKED) data.devices_status = STATUS_STARTING;
-
-    hc_thread_t i_thread = 0;
-
-    if ((data.wordlist_mode == WL_MODE_FILE) || (data.wordlist_mode == WL_MODE_MASK))
+    if ((data.devices_status != STATUS_BYPASS) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
     {
-      hc_thread_create (i_thread, thread_keypress, &benchmark);
+      data.devices_status = STATUS_STARTING;
     }
 
-    if (wordlist_mode == WL_MODE_STDIN) data.status = 1;
-
-    uint ni_threads_cnt = 0;
+    uint inner_threads_cnt = 0;
 
-    hc_thread_t *ni_threads = (hc_thread_t *) mycalloc (10, sizeof (hc_thread_t));
+    hc_thread_t *inner_threads = (hc_thread_t *) mycalloc (10, sizeof (hc_thread_t));
 
-    hc_thread_create (ni_threads[ni_threads_cnt], thread_monitor, NULL);
-
-    ni_threads_cnt++;
+    data.shutdown_inner = 0;
 
     /**
       * Outfile remove
       */
 
-    if (keyspace == 0)
+    if (keyspace == 0 && benchmark == 0 && stdout_flag == 0)
     {
+      hc_thread_create (inner_threads[inner_threads_cnt], thread_monitor, NULL);
+
+      inner_threads_cnt++;
+
       if (outfile_check_timer != 0)
       {
         if (data.outfile_check_directory != NULL)
@@ -16657,9 +17312,9 @@ int main (int argc, char **argv)
               !((hash_mode >= 13700) && (hash_mode <= 13799)) &&
               (hash_mode != 9000))
           {
-            hc_thread_create (ni_threads[ni_threads_cnt], thread_outfile_remove, NULL);
+            hc_thread_create (inner_threads[inner_threads_cnt], thread_outfile_remove, NULL);
 
-            ni_threads_cnt++;
+            inner_threads_cnt++;
           }
           else
           {
@@ -16681,8 +17336,8 @@ int main (int argc, char **argv)
     {
       if (potfile_remove_cracks > 0)
       {
-        if (potfile_remove_cracks == 1) log_info ("INFO: removed 1 hash found in pot file\n");
-        else                            log_info ("INFO: removed %u hashes found in pot file\n", potfile_remove_cracks);
+        if (potfile_remove_cracks == 1) log_info ("INFO: Removed 1 hash found in pot file\n");
+        else                            log_info ("INFO: Removed %u hashes found in pot file\n", potfile_remove_cracks);
       }
     }
 
@@ -16705,9 +17360,9 @@ int main (int argc, char **argv)
 
     for (uint maskpos = rd->maskpos; maskpos < maskcnt; maskpos++)
     {
-      if (data.devices_status == STATUS_CRACKED) break;
-
-      data.devices_status = STATUS_INIT;
+      if (data.devices_status == STATUS_CRACKED) continue;
+      if (data.devices_status == STATUS_ABORTED) continue;
+      if (data.devices_status == STATUS_QUIT)    continue;
 
       if (maskpos > rd->maskpos)
       {
@@ -16946,15 +17601,24 @@ int main (int argc, char **argv)
         }
       }
 
-      for (uint dictpos = rd->dictpos; dictpos < dictcnt; )
+      for (uint dictpos = rd->dictpos; dictpos < dictcnt; dictpos++)
       {
+        if (data.devices_status == STATUS_CRACKED) continue;
+        if (data.devices_status == STATUS_ABORTED) continue;
+        if (data.devices_status == STATUS_QUIT)    continue;
+
+        rd->dictpos = dictpos;
+
         char *subid = logfile_generate_subid ();
 
         data.subid = subid;
 
         logfile_sub_msg ("START");
 
-        data.devices_status = STATUS_INIT;
+        if ((data.devices_status != STATUS_BYPASS) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
+        {
+          data.devices_status = STATUS_INIT;
+        }
 
         memset (data.words_progress_done,     0, data.salts_cnt * sizeof (u64));
         memset (data.words_progress_rejected, 0, data.salts_cnt * sizeof (u64));
@@ -16979,7 +17643,7 @@ int main (int argc, char **argv)
 
         data.ms_paused = 0;
 
-        data.kernel_power_div = 0;
+        data.kernel_power_final = 0;
 
         data.words_cur = rd->words_cur;
 
@@ -17054,10 +17718,7 @@ int main (int argc, char **argv)
 
             if (data.words_cnt == 0)
             {
-              if (data.devices_status == STATUS_CRACKED) break;
-              if (data.devices_status == STATUS_ABORTED) break;
-
-              dictpos++;
+              logfile_sub_msg ("STOP");
 
               continue;
             }
@@ -17104,10 +17765,7 @@ int main (int argc, char **argv)
 
           if (data.words_cnt == 0)
           {
-            if (data.devices_status == STATUS_CRACKED) break;
-            if (data.devices_status == STATUS_ABORTED) break;
-
-            dictpos++;
+            logfile_sub_msg ("STOP");
 
             continue;
           }
@@ -17147,10 +17805,7 @@ int main (int argc, char **argv)
 
           if (data.words_cnt == 0)
           {
-            if (data.devices_status == STATUS_CRACKED) break;
-            if (data.devices_status == STATUS_ABORTED) break;
-
-            dictpos++;
+            logfile_sub_msg ("STOP");
 
             continue;
           }
@@ -17204,20 +17859,16 @@ int main (int argc, char **argv)
           {
             if (css_cnt < mask_min)
             {
-              log_info ("WARNING: skipping mask '%s' because it is smaller than the minimum password length", mask);
+              log_info ("WARNING: Skipping mask '%s' because it is smaller than the minimum password length", mask);
             }
 
             if (css_cnt > mask_max)
             {
-              log_info ("WARNING: skipping mask '%s' because it is larger than the maximum password length", mask);
+              log_info ("WARNING: Skipping mask '%s' because it is larger than the maximum password length", mask);
             }
 
             // skip to next mask
 
-            dictpos++;
-
-            rd->dictpos = dictpos;
-
             logfile_sub_msg ("STOP");
 
             continue;
@@ -17424,7 +18075,7 @@ int main (int argc, char **argv)
 
         if (data.words_cur > data.words_base)
         {
-          log_error ("ERROR: restore value greater keyspace");
+          log_error ("ERROR: Restore value greater keyspace");
 
           return (-1);
         }
@@ -17489,37 +18140,17 @@ int main (int argc, char **argv)
           }
         }
 
-        data.devices_status = STATUS_RUNNING;
-
-        if (initial_restore_done == 0)
-        {
-          if (data.restore_disable == 0) cycle_restore ();
-
-          initial_restore_done = 1;
-        }
-
-        hc_timer_set (&data.timer_running);
-
-        if ((wordlist_mode == WL_MODE_FILE) || (wordlist_mode == WL_MODE_MASK))
-        {
-          if ((quiet == 0) && (status == 0) && (benchmark == 0))
-          {
-            if (quiet == 0) fprintf (stdout, "%s", PROMPT);
-            if (quiet == 0) fflush (stdout);
-          }
-        }
-        else if (wordlist_mode == WL_MODE_STDIN)
-        {
-          if (data.quiet == 0) log_info ("Starting attack in stdin mode...");
-          if (data.quiet == 0) log_info ("");
-        }
-
         /**
          * create autotune threads
          */
 
         hc_thread_t *c_threads = (hc_thread_t *) mycalloc (data.devices_cnt, sizeof (hc_thread_t));
 
+        if ((data.devices_status != STATUS_BYPASS) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
+        {
+          data.devices_status = STATUS_AUTOTUNE;
+        }
+
         for (uint device_id = 0; device_id < data.devices_cnt; device_id++)
         {
           hc_device_param_t *device_param = &devices_param[device_id];
@@ -17533,15 +18164,21 @@ int main (int argc, char **argv)
          * Inform user about possible slow speeds
          */
 
+        uint hardware_power_all = 0;
+
         uint kernel_power_all = 0;
 
         for (uint device_id = 0; device_id < data.devices_cnt; device_id++)
         {
           hc_device_param_t *device_param = &devices_param[device_id];
 
+          hardware_power_all += device_param->hardware_power;
+
           kernel_power_all += device_param->kernel_power;
         }
 
+        data.hardware_power_all = hardware_power_all; // hardware_power_all is the same as kernel_power_all but without the influence of kernel_accel on the devices
+
         data.kernel_power_all = kernel_power_all;
 
         if ((wordlist_mode == WL_MODE_FILE) || (wordlist_mode == WL_MODE_MASK))
@@ -17550,6 +18187,8 @@ int main (int argc, char **argv)
           {
             if (quiet == 0)
             {
+              clear_prompt ();
+
               log_info ("ATTENTION!");
               log_info ("  The wordlist or mask you are using is too small.");
               log_info ("  Therefore, hashcat is unable to utilize the full parallelization power of your device(s).");
@@ -17564,6 +18203,34 @@ int main (int argc, char **argv)
          * create cracker threads
          */
 
+        if ((data.devices_status != STATUS_BYPASS) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
+        {
+          data.devices_status = STATUS_RUNNING;
+        }
+
+        if (initial_restore_done == 0)
+        {
+          if (data.restore_disable == 0) cycle_restore ();
+
+          initial_restore_done = 1;
+        }
+
+        hc_timer_set (&data.timer_running);
+
+        if ((wordlist_mode == WL_MODE_FILE) || (wordlist_mode == WL_MODE_MASK))
+        {
+          if ((quiet == 0) && (status == 0) && (benchmark == 0))
+          {
+            if (quiet == 0) fprintf (stdout, "%s", PROMPT);
+            if (quiet == 0) fflush (stdout);
+          }
+        }
+        else if (wordlist_mode == WL_MODE_STDIN)
+        {
+          if (data.quiet == 0) log_info ("Starting attack in stdin mode...");
+          if (data.quiet == 0) log_info ("");
+        }
+
         time_t runtime_start;
 
         time (&runtime_start);
@@ -17588,21 +18255,14 @@ int main (int argc, char **argv)
 
         local_free (c_threads);
 
-        data.restore = 0;
-
-        // finalize task
+        if ((data.devices_status != STATUS_BYPASS) && (data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
+        {
+          data.devices_status = STATUS_EXHAUSTED;
+        }
 
         logfile_sub_var_uint ("status-after-work", data.devices_status);
 
-        if (data.devices_status == STATUS_STOP_AT_CHECKPOINT) check_checkpoint ();
-
-        if (data.devices_status == STATUS_CRACKED) break;
-        if (data.devices_status == STATUS_ABORTED) break;
-
-        if (data.devices_status == STATUS_BYPASS)
-        {
-          data.devices_status = STATUS_RUNNING;
-        }
+        data.restore = 0;
 
         if (induction_dictionaries_cnt)
         {
@@ -17618,47 +18278,34 @@ int main (int argc, char **argv)
           induction_dictionaries_cnt = count_dictionaries (induction_dictionaries);
         }
 
-        if (benchmark == 0)
+        if (benchmark == 1)
         {
-          if (((dictpos + 1) < dictcnt) || ((maskpos + 1) < maskcnt) || induction_dictionaries_cnt)
-          {
-            if (quiet == 0) clear_prompt ();
-
-            if (quiet == 0) log_info ("");
-
-            if (status == 1)
-            {
-              status_display ();
-            }
-            else
-            {
-              if (quiet == 0) status_display ();
-            }
+          status_benchmark ();
 
-            if (quiet == 0) log_info ("");
+          if (machine_readable == 0)
+          {
+            log_info ("");
           }
         }
-
-        if (attack_mode == ATTACK_MODE_BF)
-        {
-          dictpos++;
-
-          rd->dictpos = dictpos;
-        }
         else
         {
-          if (induction_dictionaries_cnt)
+          if (quiet == 0)
           {
-            qsort (induction_dictionaries, induction_dictionaries_cnt, sizeof (char *), sort_by_mtime);
-          }
-          else
-          {
-            dictpos++;
+            clear_prompt ();
 
-            rd->dictpos = dictpos;
+            log_info ("");
+
+            if (stdout_flag == 0) status_display ();
+
+            log_info ("");
           }
         }
 
+        if (induction_dictionaries_cnt)
+        {
+          qsort (induction_dictionaries, induction_dictionaries_cnt, sizeof (char *), sort_by_mtime);
+        }
+
         time_t runtime_stop;
 
         time (&runtime_stop);
@@ -17671,22 +18318,37 @@ int main (int argc, char **argv)
         logfile_sub_msg ("STOP");
 
         global_free (subid);
-      }
 
-      if (data.devices_status == STATUS_STOP_AT_CHECKPOINT) check_checkpoint ();
+        // from this point we handle bypass as running
+
+        if (data.devices_status == STATUS_BYPASS)
+        {
+          data.devices_status = STATUS_RUNNING;
+        }
+
+        // and overwrite benchmark aborts as well
+
+        if (data.benchmark == 1)
+        {
+          if (data.devices_status == STATUS_ABORTED)
+          {
+            data.devices_status = STATUS_RUNNING;
+          }
+        }
+
+        // finalize task
+
+        if (data.devices_status == STATUS_CRACKED) break;
+        if (data.devices_status == STATUS_ABORTED) break;
+        if (data.devices_status == STATUS_QUIT)    break;
+      }
 
       if (data.devices_status == STATUS_CRACKED) break;
       if (data.devices_status == STATUS_ABORTED) break;
       if (data.devices_status == STATUS_QUIT)    break;
-
-      if (data.devices_status == STATUS_BYPASS)
-      {
-        data.devices_status = STATUS_RUNNING;
-      }
     }
 
     // problems could occur if already at startup everything was cracked (because of .pot file reading etc), we must set some variables here to avoid NULL pointers
-
     if (attack_mode == ATTACK_MODE_STRAIGHT)
     {
       if (data.wordlist_mode == WL_MODE_FILE)
@@ -17725,11 +18387,6 @@ int main (int argc, char **argv)
       }
     }
 
-    if ((data.devices_status != STATUS_CRACKED) && (data.devices_status != STATUS_ABORTED) && (data.devices_status != STATUS_QUIT))
-    {
-      data.devices_status = STATUS_EXHAUSTED;
-    }
-
     // if cracked / aborted remove last induction dictionary
 
     for (int file_pos = 0; file_pos < induction_dictionaries_cnt; file_pos++)
@@ -17742,22 +18399,17 @@ int main (int argc, char **argv)
       }
     }
 
-    // wait for non-interactive threads
-
-    for (uint thread_idx = 0; thread_idx < ni_threads_cnt; thread_idx++)
-    {
-      hc_thread_wait (1, &ni_threads[thread_idx]);
-    }
-
-    local_free (ni_threads);
+    // wait for inner threads
 
-    // wait for interactive threads
+    data.shutdown_inner = 1;
 
-    if ((data.wordlist_mode == WL_MODE_FILE) || (data.wordlist_mode == WL_MODE_MASK))
+    for (uint thread_idx = 0; thread_idx < inner_threads_cnt; thread_idx++)
     {
-      hc_thread_wait (1, &i_thread);
+      hc_thread_wait (1, &inner_threads[thread_idx]);
     }
 
+    local_free (inner_threads);
+
     // we dont need restore file anymore
     if (data.restore_disable == 0)
     {
@@ -17783,33 +18435,6 @@ int main (int argc, char **argv)
      * Clean up
      */
 
-    if (benchmark == 1)
-    {
-      status_benchmark ();
-
-      if (machine_readable == 0)
-      {
-        log_info ("");
-      }
-    }
-    else
-    {
-      if (quiet == 0) clear_prompt ();
-
-      if (quiet == 0) log_info ("");
-
-      if (status == 1)
-      {
-        status_display ();
-      }
-      else
-      {
-        if (quiet == 0) status_display ();
-      }
-
-      if (quiet == 0) log_info ("");
-    }
-
     for (uint device_id = 0; device_id < data.devices_cnt; device_id++)
     {
       hc_device_param_t *device_param = &data.devices_param[device_id];
@@ -17817,15 +18442,10 @@ int main (int argc, char **argv)
       if (device_param->skipped) continue;
 
       local_free (device_param->combs_buf);
-
       local_free (device_param->hooks_buf);
-
       local_free (device_param->device_name);
-
       local_free (device_param->device_name_chksum);
-
       local_free (device_param->device_version);
-
       local_free (device_param->driver_version);
 
       if (device_param->pws_buf)            myfree                    (device_param->pws_buf);
@@ -17853,7 +18473,10 @@ int main (int argc, char **argv)
       if (device_param->d_tmps)             hc_clReleaseMemObject     (data.ocl, device_param->d_tmps);
       if (device_param->d_hooks)            hc_clReleaseMemObject     (data.ocl, device_param->d_hooks);
       if (device_param->d_result)           hc_clReleaseMemObject     (data.ocl, device_param->d_result);
-      if (device_param->d_scryptV_buf)      hc_clReleaseMemObject     (data.ocl, device_param->d_scryptV_buf);
+      if (device_param->d_scryptV0_buf)     hc_clReleaseMemObject     (data.ocl, device_param->d_scryptV0_buf);
+      if (device_param->d_scryptV1_buf)     hc_clReleaseMemObject     (data.ocl, device_param->d_scryptV1_buf);
+      if (device_param->d_scryptV2_buf)     hc_clReleaseMemObject     (data.ocl, device_param->d_scryptV2_buf);
+      if (device_param->d_scryptV3_buf)     hc_clReleaseMemObject     (data.ocl, device_param->d_scryptV3_buf);
       if (device_param->d_root_css_buf)     hc_clReleaseMemObject     (data.ocl, device_param->d_root_css_buf);
       if (device_param->d_markov_css_buf)   hc_clReleaseMemObject     (data.ocl, device_param->d_markov_css_buf);
       if (device_param->d_tm_c)             hc_clReleaseMemObject     (data.ocl, device_param->d_tm_c);
@@ -17883,7 +18506,7 @@ int main (int argc, char **argv)
     #ifdef HAVE_HWMON
     if (gpu_temp_disable == 0)
     {
-      if (gpu_temp_retain != 0) // VENDOR_ID_AMD is implied here
+      if (gpu_temp_retain != 0)
       {
         hc_thread_mutex_lock (mux_adl);
 
@@ -17895,24 +18518,24 @@ int main (int argc, char **argv)
 
           if (data.hm_device[device_id].fan_set_supported == 1)
           {
-            int fanspeed  = temp_retain_fanspeed_value[device_id];
-            int fanpolicy = temp_retain_fanpolicy_value[device_id];
+            int rc = -1;
 
-            if (fanpolicy == 1)
+            if (device_param->device_vendor_id == VENDOR_ID_AMD)
             {
-              int rc = -1;
-
-              if (device_param->device_vendor_id == VENDOR_ID_AMD)
-              {
-                rc = hm_set_fanspeed_with_device_id_adl (device_id, fanspeed, 0);
-              }
-              else if (device_param->device_vendor_id == VENDOR_ID_NV)
-              {
-
-              }
+              rc = hm_set_fanspeed_with_device_id_adl (device_id, 100, 0);
+            }
+            else if (device_param->device_vendor_id == VENDOR_ID_NV)
+            {
+              #ifdef LINUX
+              rc = set_fan_control (data.hm_xnvctrl, data.hm_device[device_id].xnvctrl, NV_CTRL_GPU_COOLER_MANUAL_CONTROL_FALSE);
+              #endif
 
-              if (rc == -1) log_info ("WARNING: Failed to restore default fan speed and policy for device #%", device_id + 1);
+              #ifdef WIN
+              rc = hm_set_fanspeed_with_device_id_nvapi (device_id, 100, 0);
+              #endif
             }
+
+            if (rc == -1) log_info ("WARNING: Failed to restore default fan speed and policy for device #%", device_id + 1);
           }
         }
 
@@ -17922,7 +18545,7 @@ int main (int argc, char **argv)
 
     // reset power tuning
 
-    if (powertune_enable == 1) // VENDOR_ID_AMD is implied here
+    if (powertune_enable == 1)
     {
       hc_thread_mutex_lock (mux_adl);
 
@@ -18006,6 +18629,24 @@ int main (int argc, char **argv)
         data.hm_nvml = NULL;
       }
 
+      if (data.hm_nvapi)
+      {
+        hm_NvAPI_Unload (data.hm_nvapi);
+
+        nvapi_close (data.hm_nvapi);
+
+        data.hm_nvapi = NULL;
+      }
+
+      if (data.hm_xnvctrl)
+      {
+        hm_XNVCTRL_XCloseDisplay (data.hm_xnvctrl);
+
+        xnvctrl_close (data.hm_xnvctrl);
+
+        data.hm_xnvctrl = NULL;
+      }
+
       if (data.hm_adl)
       {
         hm_ADL_Main_Control_Destroy (data.hm_adl);
@@ -18055,7 +18696,6 @@ int main (int argc, char **argv)
     local_free (bitmap_s2_d);
 
     #ifdef HAVE_HWMON
-    local_free (temp_retain_fanspeed_value);
     local_free (od_clock_mem_status);
     local_free (od_power_control_status);
     local_free (nvml_power_limit);
@@ -18086,6 +18726,17 @@ int main (int argc, char **argv)
     if (data.devices_status == STATUS_QUIT) break;
   }
 
+  // wait for outer threads
+
+  data.shutdown_outer = 1;
+
+  for (uint thread_idx = 0; thread_idx < outer_threads_cnt; thread_idx++)
+  {
+    hc_thread_wait (1, &outer_threads[thread_idx]);
+  }
+
+  local_free (outer_threads);
+
   // destroy others mutex
 
   hc_thread_mutex_delete (mux_dispatcher);