Set annotation featureRestartCount (#289)

The annotation featureRestartCount is set to "true" if the restart count
in the container status can be set properly. This is the case if systemd
version 235 or newer is used. If an older systemd version is used then
featureRestartCount is set to "false".

Author: siegfriedweber

CHANGELOG.md

@@ -7,6 +7,8 @@
 - Check added on startup if the configured directories exist and are
   writable by the Stackable agent ([#273]).
 - Missing directories are created ([#274]).
+- Annotation `featureRestartCount` added to the pods to indicate if the
+  restart count is set properly ([#289]).
 
 ### Changed
 - Lazy validation of repository URLs changed to eager validation
@@ -35,6 +37,7 @@
 [#273]: https://github.com/stackabletech/agent/pull/273
 [#274]: https://github.com/stackabletech/agent/pull/274
 [#276]: https://github.com/stackabletech/agent/pull/276
+[#289]: https://github.com/stackabletech/agent/pull/289
 
 ## 0.5.0 - 2021-07-26
 

Cargo.lock

@@ -50,8 +50,8 @@ tar = "0.4"
 thiserror = "1.0"
 tokio = { version = "1.11", features = ["macros", "rt-multi-thread", "time"] }
 url = "2.2"
-zbus = { git = "https://gitlab.freedesktop.org/dbus/zbus", rev = "0a8a4268be0f844ab4bca429f27766347dc3af58" } # version 2.0.0-beta.6 + merge request !354 (fixes a race condition)
-zvariant = { git = "https://gitlab.freedesktop.org/dbus/zbus", rev = "0a8a4268be0f844ab4bca429f27766347dc3af58" } # version 2.8.0 which is compatible with the zbus version
+zbus = { git = "https://gitlab.freedesktop.org/dbus/zbus", rev = "ff08cbbbcd3eead16464012b92e3862d4dcb6f16" } # version 2.0.0-beta.6 + merge request !354 (fixes a race condition) + commit 6cdfe48cda5e0bf7b0dd8675be7a84439678afa9 (fixes another race condition)
+zvariant = { git = "https://gitlab.freedesktop.org/dbus/zbus", rev = "ff08cbbbcd3eead16464012b92e3862d4dcb6f16" } # version 2.8.0 which is compatible with the zbus version
 
 [dev-dependencies]
 indoc = "1.0"

Cargo.toml

@@ -0,0 +1,17 @@
+= Restarts
+
+The restart count is stored in the container status if systemd version
+235 or newer is running on the node which is the case for Debian 10 and
+CentOS 8 but not for CentOS 7. The annotation `featureRestartCount`
+indicates whether or not the restart count is set properly.
+
+    $ kubectl get pod <pod-name>
+    NAME         READY   STATUS    RESTARTS   AGE
+    <pod-name>   1/1     Running   4          10m
+
+    $ kubectl describe pod <pod-name>
+    Name:         <pod-name>
+    Annotations:  featureRestartCount: true
+    Containers:
+      <service-name>:
+        Restart Count:  4

docs/modules/ROOT/pages/monitoring/restarts.adoc

@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+
 use super::running::Running;
 use crate::provider::{
     kubernetes::status::patch_container_status, systemdmanager::service::ServiceState, PodHandle,
@@ -79,41 +81,46 @@ async fn start_service_units(
             );
         }
 
-        add_annotation(
-            &client,
-            pod,
+        let mut annotations = HashMap::new();
+        annotations.insert(
             "featureLogs",
-            &systemd_service.invocation_id().await.is_ok().to_string(),
-        )
-        .await?;
+            systemd_service.invocation_id().await.is_ok().to_string(),
+        );
+        annotations.insert(
+            "featureRestartCount",
+            systemd_service.restart_count().await.is_ok().to_string(),
+        );
+
+        add_annotations(&client, pod, &annotations).await?;
 
         patch_container_status(&client, pod, &container_key, &Status::running()).await;
     }
 
     Ok(())
 }
 
-/// Adds an annotation to the given pod.
+/// Adds annotations to the given pod.
 ///
 /// If there is already an annotation with the given key then the value
 /// is replaced.
 /// The function returns when the patch is sent. It does not await the
 /// changes to be visible to the watching clients.
-async fn add_annotation(client: &Client, pod: &Pod, key: &str, value: &str) -> kube::Result<Pod> {
+async fn add_annotations(
+    client: &Client,
+    pod: &Pod,
+    annotations: &HashMap<&str, String>,
+) -> kube::Result<Pod> {
     debug!(
-        "Adding annotation [{}: {}] to pod [{:?}]",
-        key,
-        value,
+        "Adding annotations [{:?}] to pod [{:?}]",
+        annotations,
         PodKey::from(pod)
     );
 
     let api: Api<Pod> = Api::namespaced(client.clone(), pod.namespace());
 
     let patch = json!({
         "metadata": {
-            "annotations": {
-                key: value
-            }
+            "annotations": annotations
         }
     });
 

src/provider/states/pod/starting.rs

@@ -112,6 +112,9 @@ impl SystemdService {
         Ok(service_state)
     }
 
+    /// Retrieves the current restart count.
+    ///
+    /// The restart counter was introduced in systemd version 235.
     pub async fn restart_count(&self) -> anyhow::Result<u32> {
         self.service_proxy
             .nrestarts()