Merge pull request #468 from djoshy/add-mc-logs

NO-ISSUE: gather additional MCO debug artifacts

Author: openshift-merge-bot[bot]

collection-scripts/gather

@@ -142,6 +142,10 @@ pids+=($!)
 /usr/bin/gather_machineconfig_ondisk &
 pids+=($!)
 
+# Gather On-Disk MachineConfigDaemon logs
+/usr/bin/gather_machineconfigdaemon_termination_logs &
+pids+=($!)
+
 # Gather vSphere resources. This is NOOP on non-vSphere platform.
 /usr/bin/gather_vsphere &
 pids+=($!)

collection-scripts/gather_machineconfig_ondisk

@@ -16,6 +16,7 @@ function gather_machineconfig_ondisk {
          DAEMONPOD=$(oc get pods -n openshift-machine-config-operator --field-selector spec.nodeName=${NODE} --selector k8s-app=machine-config-daemon -o custom-columns=:metadata.name --no-headers)
          # collect the boostrap config that the Node got from the MCS
          timeout -v 1m oc cp -c machine-config-daemon openshift-machine-config-operator/"${DAEMONPOD}":rootfs/etc/mcs-machine-config-content.json "${MACHINECONFIG_CONFIG_PATH}"/"${NODE}"/mcs-machine-config-content.json &
+         timeout -v 1m oc cp -c machine-config-daemon openshift-machine-config-operator/"${DAEMONPOD}":rootfs/etc/machine-config-daemon/bootstrapconfigdiff "${MACHINECONFIG_CONFIG_PATH}"/"${NODE}"/bootstrapconfigdiff &
          PIDS+=($!)
     done
 

collection-scripts/gather_machineconfigdaemon_termination_logs

@@ -0,0 +1,38 @@
+#!/bin/bash 
+
+BASE_COLLECTION_PATH="must-gather"
+MACHINECONFIG_DAEMON_LOG_PATH=${OUT:-"${BASE_COLLECTION_PATH}/machine_config_termination_logs"}
+
+mkdir -p "${MACHINECONFIG_DAEMON_LOG_PATH}"/
+
+# gather_machineconfigdaemon_termination_logs collects the daemon logs before it was last terminated
+function gather_machineconfigdaemon_termination_logs {
+     
+    echo "INFO: Gathering machine config daemon's old logs from all nodes"
+    NODES=$(oc get nodes -o jsonpath='{.items[?(@.kind=="Node")].metadata.name}')
+    for NODE in ${NODES}; do
+         # use the existing MCD pod on the node to get the files
+         DAEMONPOD=$(oc get pods -n openshift-machine-config-operator --field-selector spec.nodeName=${NODE} --selector k8s-app=machine-config-daemon -o custom-columns=:metadata.name --no-headers)
+         # Check if the 'previous-logs' folder exists on the MCD pod
+         FOLDER_EXISTS=$(oc exec -n openshift-machine-config-operator "${DAEMONPOD}" -c machine-config-daemon -- sh -c '[ -d "/rootfs/etc/machine-config-daemon/previous-logs" ] && echo "exists"' 2>/dev/null)
+         # collect the logs from the location that MCD saves to prior to shutdown
+         if [ "${FOLDER_EXISTS}" == "exists" ]; then
+           timeout -v 1m oc cp -c machine-config-daemon openshift-machine-config-operator/"${DAEMONPOD}":rootfs/etc/machine-config-daemon/previous-logs "${MACHINECONFIG_DAEMON_LOG_PATH}"/"${NODE}"/ &
+           PIDS+=($!)
+         else
+           echo "INFO: 'previous-logs' folder not found on ${NODE}, skipping..."
+         fi
+    done
+
+}
+
+PIDS=()
+gather_machineconfigdaemon_termination_logs
+
+echo "INFO: Waiting for Machine Config Daemon termination log collection to complete ..."
+wait "${PIDS[@]}"
+echo "INFO: Machine Config Daemon termination log collection complete."
+
+# force disk flush to ensure that all data gathered is accessible in the copy container
+sync
+

collection-scripts/gather_ppc

@@ -190,7 +190,7 @@ resources=()
 resources+=(performanceprofile)
 
 # machine/node resources
-resources+=(nodes machineconfigs machineconfigpools featuregates kubeletconfigs tuneds runtimeclasses machineosconfigs machineosbuilds)
+resources+=(nodes machineconfigs machineconfigpools featuregates kubeletconfigs tuneds runtimeclasses machineosconfigs machineosbuilds machineconfigurations)
 
 echo "INFO: Waiting for node performance related collection to complete ..."