Update to release 24.05.7

META

@@ -7,8 +7,8 @@
   Name:		slurm
   Major:	24
   Minor:	05
-  Micro:	6
-  Version:	24.05.6
+  Micro:	7
+  Version:	24.05.7
   Release:	1
 
 ##

NEWS

@@ -1,6 +1,17 @@
 This file describes changes in recent versions of Slurm. It primarily
 documents those changes that are of interest to users and administrators.
 
+* Changes in Slurm 24.05.7
+==========================
+ -- Fix slurmctld crash when after updating a reservation with an empty
+    nodelist. The crash could occur after restarting slurmctld, or if
+    downing/draining a node in the reservation with the REPLACE or REPLACE_DOWN
+    flag.
+ -- Fix jobs being scheduled on higher weighted powered down
+    nodes.
+ -- Fix memory leak when RestrictedCoresPerGPU is enabled.
+ -- Prevent slurmctld deadlock in the assoc mgr.
+
 * Changes in Slurm 24.05.6
 ==========================
  -- data_parser/v0.0.40 - Prevent a segfault in the slurmrestd when

debian/changelog

@@ -1,4 +1,4 @@
-slurm-smd (24.05.6-1) UNRELEASED; urgency=medium
+slurm-smd (24.05.7-1) UNRELEASED; urgency=medium
 
   * Initial release.
 

doc/html/rest_api.shtml

@@ -3,7 +3,7 @@
     <div class="app-desc">API to access and control Slurm</div>
     <div class="app-desc">More information: <a href="https://www.schedmd.com/">https://www.schedmd.com/</a></div>
     <div class="app-desc">Contact Info: <a href="[email protected]">[email protected]</a></div>
-    <div class="app-desc">Version: Slurm-24.05.6&amp;openapi/slurmdbd&amp;openapi/slurmctld</div>
+    <div class="app-desc">Version: Slurm-24.05.7&amp;openapi/slurmdbd&amp;openapi/slurmctld</div>
     <div class="app-desc">BasePath:</div>
     <div class="license-info">Apache 2.0</div>
     <div class="license-url">https://www.apache.org/licenses/LICENSE-2.0.html</div>

slurm.spec

@@ -1,5 +1,5 @@
 Name:		slurm
-Version:	24.05.6
+Version:	24.05.7
 %define rel	1
 Release:        %{rel}.%{gittag}%{?dist}%{?gpu}.ug
 Summary:	Slurm Workload Manager

src/common/assoc_mgr.c

@@ -3328,7 +3328,8 @@ extern bool assoc_mgr_is_user_acct_coord(void *db_conn,
 	if (!is_locked)
 		assoc_mgr_lock(&locks);
 	if (!assoc_mgr_coord_list || !list_count(assoc_mgr_coord_list)) {
-		assoc_mgr_unlock(&locks);
+		if (!is_locked)
+			assoc_mgr_unlock(&locks);
 		return false;
 	}
 

src/plugins/select/cons_tres/gres_sock_list.c

@@ -677,6 +677,12 @@ static void _pick_restricted_cores(bitstr_t *core_bitmap,
 		gres_js->res_gpu_cores = xcalloc(gres_js->res_array_size,
 						 sizeof(bitstr_t *));
 	}
+	/*
+	 * This function can be called multiple times for the same node_i while
+	 * a job is pending. Free any existing gres_js->res_gpu_cores[node_i]
+	 * first.
+	 */
+	FREE_NULL_BITMAP(gres_js->res_gpu_cores[node_i]);
 	gres_js->res_gpu_cores[node_i] = bit_alloc(bit_size(core_bitmap));
 
 	for (int i = 0; i < gres_ns->topo_cnt; i++) {

src/slurmctld/job_mgr.c

@@ -4742,7 +4742,7 @@ static void _apply_signal_jobs_filter(job_record_t *job_ptr,
 
 	/* Verify that the user can kill the requested job */
 	if ((job_ptr->user_id != auth_uid) &&
-	    !validate_operator(auth_uid) &&
+	    !validate_operator_locked(auth_uid) &&
 	    !assoc_mgr_is_user_acct_coord(acct_db_conn, auth_uid,
 					  job_ptr->account, true)) {
 		slurm_selected_step_t *use_id;

src/slurmctld/node_scheduler.c

@@ -4147,6 +4147,8 @@ static int _build_node_list(job_record_t *job_ptr,
 		node_set_ptr[node_set_inx].node_cnt = power_cnt;
 		node_set_ptr[i].node_cnt -= power_cnt;
 		node_set_ptr[node_set_inx].flags = NODE_SET_POWER_DN;
+		node_set_ptr[node_set_inx].node_weight =
+			node_set_ptr[i].node_weight;
 		node_set_ptr[node_set_inx].features =
 			xstrdup(node_set_ptr[i].features);
 		node_set_ptr[node_set_inx].feature_bits =

src/slurmctld/proc_req.c

@@ -575,18 +575,37 @@ extern bool validate_super_user(uid_t uid)
  * IN uid - user to validate
  * RET true if permitted to run, false otherwise
  */
-extern bool validate_operator(uid_t uid)
+static bool _validate_operator_internal(uid_t uid, bool locked)
 {
+	slurmdb_admin_level_t level;
+
 #ifndef NDEBUG
 	if (drop_priv)
 		return false;
 #endif
-	if ((uid == 0) || (uid == slurm_conf.slurm_user_id) ||
-	    assoc_mgr_get_admin_level(acct_db_conn, uid) >=
-	    SLURMDB_ADMIN_OPERATOR)
+
+	if ((uid == 0) || (uid == slurm_conf.slurm_user_id))
 		return true;
+
+	if (locked)
+		level = assoc_mgr_get_admin_level_locked(acct_db_conn, uid);
 	else
-		return false;
+		level = assoc_mgr_get_admin_level(acct_db_conn, uid);
+
+	if (level >= SLURMDB_ADMIN_OPERATOR)
+		return true;
+
+	return false;
+}
+
+extern bool validate_operator(uid_t uid)
+{
+	return _validate_operator_internal(uid, false);
+}
+
+extern bool validate_operator_locked(uid_t uid)
+{
+	return _validate_operator_internal(uid, true);
 }
 
 extern bool validate_operator_user_rec(slurmdb_user_rec_t *user)

src/slurmctld/reservation.c

@@ -3343,6 +3343,19 @@ static int _validate_reservation_access_update(void *x, void *y)
 	return 0;
 }
 
+static int _validate_and_set_partition(part_record_t **part_ptr,
+				       char **partition)
+{
+	if (*part_ptr == NULL) {
+		*part_ptr = default_part_loc;
+		if (*part_ptr == NULL)
+			return ESLURM_DEFAULT_PARTITION_NOT_SET;
+	}
+	xfree(*partition);
+	*partition = xstrdup((*part_ptr)->name);
+	return SLURM_SUCCESS;
+}
+
 /* Update an exiting resource reservation */
 extern int update_resv(resv_desc_msg_t *resv_desc_ptr, char **err_msg)
 {
@@ -5016,7 +5029,7 @@ extern int validate_job_resv(job_record_t *job_ptr)
 static int  _resize_resv(slurmctld_resv_t *resv_ptr, uint32_t node_cnt)
 {
 	bitstr_t *tmp2_bitmap = NULL;
-	int delta_node_cnt, i;
+	int delta_node_cnt, i, rc;
 	resv_desc_msg_t resv_desc;
 	resv_select_t resv_select = { 0 };
 
@@ -5061,10 +5074,9 @@ static int  _resize_resv(slurmctld_resv_t *resv_ptr, uint32_t node_cnt)
 	}
 
 	/* Ensure if partition exists in reservation otherwise use default */
-	if (!resv_ptr->part_ptr) {
-		resv_ptr->part_ptr = default_part_loc;
-		if (!resv_ptr->part_ptr)
-			return ESLURM_DEFAULT_PARTITION_NOT_SET;
+	if ((rc = _validate_and_set_partition(&resv_ptr->part_ptr,
+					      &resv_ptr->partition))) {
+		return rc;
 	}
 
 	/* Must increase node count. Make this look like new request so
@@ -5086,10 +5098,10 @@ static int  _resize_resv(slurmctld_resv_t *resv_ptr, uint32_t node_cnt)
 		bit_and_not(resv_select.node_bitmap, resv_ptr->node_bitmap);
 	}
 
-	i = _select_nodes(&resv_desc, &resv_ptr->part_ptr, &resv_select);
+	rc = _select_nodes(&resv_desc, &resv_ptr->part_ptr, &resv_select);
 	xfree(resv_desc.node_list);
 	xfree(resv_desc.partition);
-	if (i == SLURM_SUCCESS) {
+	if (rc == SLURM_SUCCESS) {
 		job_record_t *job_ptr = resv_desc.job_ptr;
 		/*
 		 * If the reservation was 0 node count before (ANY_NODES) this
@@ -5121,7 +5133,7 @@ static int  _resize_resv(slurmctld_resv_t *resv_ptr, uint32_t node_cnt)
 	}
 	job_record_delete(resv_desc.job_ptr);
 
-	return i;
+	return rc;
 }
 
 static int _feature_has_node_cnt(void *x, void *key)
@@ -5317,12 +5329,9 @@ static int _select_nodes(resv_desc_msg_t *resv_desc_ptr,
 	list_itr_t *itr;
 	job_record_t *job_ptr;
 
-	if (*part_ptr == NULL) {
-		*part_ptr = default_part_loc;
-		if (*part_ptr == NULL)
-			return ESLURM_DEFAULT_PARTITION_NOT_SET;
-		xfree(resv_desc_ptr->partition);	/* should be no-op */
-		resv_desc_ptr->partition = xstrdup((*part_ptr)->name);
+	if ((rc = _validate_and_set_partition(part_ptr,
+					      &resv_desc_ptr->partition))) {
+		return rc;
 	}
 
 	xassert(resv_desc_ptr->job_ptr);

src/slurmctld/slurmctld.h

@@ -2052,6 +2052,7 @@ extern bool validate_super_user(uid_t uid);
  * RET true if permitted to run, false otherwise
  */
 extern bool validate_operator(uid_t uid);
+extern bool validate_operator_locked(uid_t uid);
 
 /*
  * validate_operator_user_rec - validate that the user is authorized at the

testsuite/python/conftest.py

@@ -176,6 +176,21 @@ def module_setup(request, tmp_path_factory):
         )
         atf.stop_slurm(quiet=True)
 
+    # Cleanup StateSaveLocation for auto-config
+    if atf.properties["auto-config"]:
+        statesaveloc = atf.get_config_parameter(
+            "StateSaveLocation", live=False, quiet=True
+        )
+        if os.path.exists(statesaveloc):
+            if os.path.exists(statesaveloc + name):
+                logging.warning(
+                    f"Backup for StateSaveLocation already exists ({statesaveloc+name}). Removing it."
+                )
+                atf.run_command(f"rm -rf {statesaveloc+name}", user="root", quiet=True)
+            atf.run_command(
+                f"mv {statesaveloc} {statesaveloc+name}", user="root", quiet=True
+            )
+
     yield
 
     # Return to the folder from which pytest was executed
@@ -184,6 +199,14 @@ def module_setup(request, tmp_path_factory):
     # Teardown
     module_teardown()
 
+    # Restore StateSaveLocation for auto-config
+    if atf.properties["auto-config"]:
+        atf.run_command(f"rm -rf {statesaveloc}", user="root", quiet=True)
+        if os.path.exists(statesaveloc + name):
+            atf.run_command(
+                f"mv {statesaveloc+name} {statesaveloc}", user="root", quiet=True
+            )
+
 
 def module_teardown():
     failures = []

testsuite/python/lib/atf.py

@@ -535,6 +535,8 @@ def start_slurmctld(clean=False, quiet=False):
     if not properties["auto-config"]:
         require_auto_config("wants to start slurmctld")
 
+    logging.debug("Starting slurmctld...")
+
     if not is_slurmctld_running(quiet=quiet):
         # Start slurmctld
         command = f"{properties['slurm-sbin-dir']}/slurmctld"
@@ -550,7 +552,19 @@ def start_slurmctld(clean=False, quiet=False):
         if not repeat_command_until(
             "scontrol ping", lambda results: re.search(r"is UP", results["stdout"])
         ):
-            pytest.fail(f"Slurmctld is not running")
+            logging.warning(
+                "scontrol ping is not responding, trying to get slurmctld backtrace..."
+            )
+            pids = pids_from_exe(f"{properties['slurm-sbin-dir']}/slurmctld")
+            if not pids:
+                logging.warning("process slurmctld not found")
+            for pid in pids:
+                run_command(
+                    f'sudo gdb -p {pid} -ex "set debuginfod enabled on" -ex "set pagination off" -ex "set confirm off" -ex "set print pretty on" -ex "set max-value-size unlimited" -ex "set print array-indexes on" -ex "set print array off" -ex "thread apply all bt full" -ex "quit"'
+                )
+            pytest.fail("Slurmctld is not running")
+        else:
+            logging.debug("Slurmctld started successfully")
 
 
 def start_slurmdbd(clean=False, quiet=False):
@@ -591,7 +605,17 @@ def start_slurmdbd(clean=False, quiet=False):
         if not repeat_command_until(
             "sacctmgr show cluster", lambda results: results["exit_code"] == 0
         ):
-            pytest.fail(f"Slurmdbd is not running")
+            logging.warning(
+                "sacctmgr show cluster is not responding, trying to get slurmdbd backtrace..."
+            )
+            pids = pids_from_exe(f"{properties['slurm-sbin-dir']}/slurmdbd")
+            if not pids:
+                logging.warning("process slurmdbd not found")
+            for pid in pids:
+                run_command(
+                    f'sudo gdb -p {pid} -ex "set debuginfod enabled on" -ex "set pagination off" -ex "set confirm off" -ex "set print pretty on" -ex "set max-value-size unlimited" -ex "set print array-indexes on" -ex "set print array off" -ex "thread apply all bt full" -ex "quit"'
+                )
+            pytest.fail("Slurmdbd is not running")
         else:
             logging.debug("Slurmdbd started successfully")
 
@@ -813,7 +837,7 @@ def stop_slurm(fatal=True, quiet=False):
         logging.warning("Getting the bt of the still running slurmctld")
         for pid in pids:
             run_command(
-                f'sudo gdb -p {pid} -ex "set debuginfod enabled on" -ex "set pagination off" -ex "set confirm off" -ex "thread apply all bt" -ex "quit"'
+                f'sudo gdb -p {pid} -ex "set debuginfod enabled on" -ex "set pagination off" -ex "set confirm off" -ex "set print pretty on" -ex "set max-value-size unlimited" -ex "set print array-indexes on" -ex "set print array off" -ex "thread apply all bt full" -ex "quit"'
             )
 
     # Build list of slurmds
@@ -844,7 +868,7 @@ def stop_slurm(fatal=True, quiet=False):
         failures.append(f"Some slurmds are still running ({pids})")
         for pid in pids:
             run_command(
-                f'sudo gdb -p {pid} -ex "set debuginfod enabled on" -ex "set pagination off" -ex "set confirm off" -ex "thread apply all bt" -ex "quit"'
+                f'sudo gdb -p {pid} -ex "set debuginfod enabled on" -ex "set pagination off" -ex "set confirm off" -ex "set print pretty on" -ex "set max-value-size unlimited" -ex "set print array-indexes on" -ex "set print array off" -ex "thread apply all bt full" -ex "quit"'
             )
         run_command(f"pgrep -f {properties['slurm-sbin-dir']}/slurmd -a", quiet=quiet)
 
@@ -855,6 +879,7 @@ def stop_slurm(fatal=True, quiet=False):
             properties["slurmrestd"].wait(timeout=60)
         except:
             properties["slurmrestd"].kill()
+        properties["slurmrestd_log"].close()
 
     if failures:
         if fatal:
@@ -1869,6 +1894,13 @@ def start_slurmrestd():
     port = None
     attempts = 0
 
+    log_dir = os.path.dirname(
+        get_config_parameter("SlurmctldLogFile", live=False, quiet=True)
+    )
+    properties["slurmrestd_log"] = open(f"{log_dir}/slurmrestd.log", "w")
+    if not properties["slurmrestd_log"]:
+        pytest.fail(f"Unable to open slurmrestd log: {log_dir}/slurmrestd.log")
+
     while not port and attempts < 15:
         port = get_open_port()
         attempts += 1
@@ -1888,8 +1920,8 @@ def start_slurmrestd():
         properties["slurmrestd"] = subprocess.Popen(
             args,
             stdin=subprocess.DEVNULL,
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
+            stdout=properties["slurmrestd_log"],
+            stderr=properties["slurmrestd_log"],
         )
         s = None
 

testsuite/python/tests/test_123_1.py

@@ -39,7 +39,8 @@ def create_resv(request, node_list):
     if re.search(
         rf"(?:Nodes=)({node2})", atf.run_command_output("scontrol show res resv1")
     ):
-        node1, node2 = node_list.reverse()
+        node_list.reverse()
+        node1, node2 = node_list
 
     return [node1, node2, flag]