XXXX kmem: double-calling kmem_depot_ws_update isn't obvious
While the double-call is documented in a comment, it's not obvious what
exactly it is trying to accomplish.  The easiest way to address this is to
introduce a new function that "zeroes-out" the working set statistics to
force everything to be eligible for reaping.

@@ -18,10 +18,11 @@
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  */
 
 /*
  * Kernel memory allocator, as described in the following two papers and a
  * statement about the consolidator:

@@ -2167,10 +2168,25 @@
         cp->cache_empty.ml_min = cp->cache_empty.ml_total;
         mutex_exit(&cp->cache_depot_lock);
 }
 
 /*
+ * Set the working set statistics for cp's depot to zero.  (Everything is
+ * eligible for reaping.)
+ */
+static void
+kmem_depot_ws_zero(kmem_cache_t *cp)
+{
+        mutex_enter(&cp->cache_depot_lock);
+        cp->cache_full.ml_reaplimit = cp->cache_full.ml_total;
+        cp->cache_full.ml_min = cp->cache_full.ml_total;
+        cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_total;
+        cp->cache_empty.ml_min = cp->cache_empty.ml_total;
+        mutex_exit(&cp->cache_depot_lock);
+}
+
+/*
  * Reap all magazines that have fallen out of the depot's working set.
  */
 static void
 kmem_depot_ws_reap(kmem_cache_t *cp)
 {

@@ -3236,18 +3252,11 @@
                         kmem_magazine_destroy(cp, mp, rounds);
                 if (pmp)
                         kmem_magazine_destroy(cp, pmp, prounds);
         }
 
-        /*
-         * Updating the working set statistics twice in a row has the
-         * effect of setting the working set size to zero, so everything
-         * is eligible for reaping.
-         */
-        kmem_depot_ws_update(cp);
-        kmem_depot_ws_update(cp);
-
+        kmem_depot_ws_zero(cp);
         kmem_depot_ws_reap(cp);
 }
 
 /*
  * Enable per-cpu magazines on a cache.

@@ -3268,20 +3277,18 @@
         }
 
 }
 
 /*
- * Reap (almost) everything right now.  See kmem_cache_magazine_purge()
- * for explanation of the back-to-back kmem_depot_ws_update() calls.
+ * Reap (almost) everything right now.
  */
 void
 kmem_cache_reap_now(kmem_cache_t *cp)
 {
         ASSERT(list_link_active(&cp->cache_link));
 
-        kmem_depot_ws_update(cp);
-        kmem_depot_ws_update(cp);
+        kmem_depot_ws_zero(cp);
 
         (void) taskq_dispatch(kmem_taskq,
             (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
         taskq_wait(kmem_taskq);
 }