ch4/progress: add MPIR_CVAR_CH4_PROGRESS_THROTTLE_NUM_PAUSES

Add alternative method to throttle, e.g. sched_yield or PAUSE, and use
MPIR_CVAR_CH4_PROGRESS_THROTTLE_NUM_PAUSES to control the amount of
throttle.

Author: hzhou

src/mpid/ch4/src/ch4_progress.h

@@ -8,6 +8,8 @@
 
 #include "ch4_impl.h"
 #include "stream_workq.h"
+/* FIXME: configure check sched_yield */
+#include <sched.h>
 
 /*
 === BEGIN_MPI_T_CVAR_INFO_BLOCK ===
@@ -59,6 +61,17 @@
         When MPIR_CVAR_CH4_PROGRESS_THROTTLE=true, do throttle when minimum of MPIR_CVAR_CH4_PROGRESS_THROTTLE_MIN_PROCS
         processes enter the throttle state (no_progress_counter reach MPIR_CVAR_CH4_PROGRESS_THROTTLE_NO_PROGRESS_COUNT).
 
+    - name        : MPIR_CVAR_CH4_PROGRESS_THROTTLE_NUM_PAUSES
+      category    : CH4
+      type        : int
+      default     : 1
+      class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
+      description : >-
+        Set the number of PAUSE (or thread_yield) for each progress throttle. If set to 0 (default), it uses
+        usleep(1) for throttle.
+
 === END_MPI_T_CVAR_INFO_BLOCK ===
 */
 
@@ -175,9 +188,6 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_progress_test(MPID_Progress_state * state)
         for (int vci = 0; vci < MPIDI_global.n_vcis; vci++) {
             MPIDI_PROGRESS(vci, true);
         }
-        if (!made_progress && MPIR_CVAR_CH4_PROGRESS_THROTTLE) {
-            usleep(1);
-        }
     } else {
         for (int i = 0; i < state->vci_count; i++) {
             int vci = state->vci[i];
@@ -202,7 +212,13 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_progress_test(MPID_Progress_state * state)
         } else if (no_progress_counter > MPIR_CVAR_CH4_PROGRESS_THROTTLE_NO_PROGRESS_COUNT) {
             int throttle_id = MPIDI_progress_throttle_start();
             if (throttle_id > MPIR_CVAR_CH4_PROGRESS_THROTTLE_MIN_PROCS) {
-                usleep(1);
+                if (MPIR_CVAR_CH4_PROGRESS_THROTTLE_NUM_PAUSES == 0) {
+                    usleep(1);
+                } else {
+                    for (int i = 0; i < MPIR_CVAR_CH4_PROGRESS_THROTTLE_NUM_PAUSES; i++) {
+                        sched_yield();
+                    }
+                }
             }
         } else {
             no_progress_counter++;