Skip to content

Commit 310dfbe

Browse files
bfeshtibfeshti
authored
Merge pull request #390 from neo4j/fix/backup-logs-assertions
2 parents c78df59 + 3f354b6 commit 310dfbe

File tree

1 file changed

+210
-45
lines changed

1 file changed

+210
-45
lines changed

‎internal/integration_tests/standalone.go‎

Lines changed: 210 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ import (
1818
"math/big"
1919
"os"
2020
"os/exec"
21-
"regexp"
2221
"strings"
2322
"sync"
2423
"sync/atomic"
@@ -858,7 +857,10 @@ func InstallNeo4jBackupAzureHelmChart(t *testing.T, standaloneReleaseName model.
858857
backupReleaseName := model.NewReleaseName("standalone-backup-azure-" + TestRunIdentifier)
859858
namespace := string(standaloneReleaseName.Namespace())
860859

860+
t.Log("Starting Azure backup test")
861+
861862
t.Cleanup(func() {
863+
t.Log("Running cleanup for Azure backup test")
862864
_ = runAll(t, "helm", [][]string{
863865
{"uninstall", backupReleaseName.String(), "--wait", "--timeout", "3m", "--namespace", namespace},
864866
}, false)
@@ -879,32 +881,82 @@ func InstallNeo4jBackupAzureHelmChart(t *testing.T, standaloneReleaseName model.
879881
Type: "FULL",
880882
}
881883
helmValues.ConsistencyCheck.Database = "system"
884+
885+
t.Logf("Installing Azure backup helm chart with values: %+v", helmValues)
882886
_, err := helmClient.Install(t, backupReleaseName.String(), namespace, helmValues)
883-
assert.NoError(t, err)
887+
if err != nil {
888+
t.Logf("Failed to install Azure backup helm chart: %v", err)
889+
return err
890+
}
884891

892+
t.Log("Waiting for Azure backup job to complete")
885893
time.Sleep(2 * time.Minute)
894+
886895
cronjob, err := Clientset.BatchV1().CronJobs(namespace).Get(context.Background(), backupReleaseName.String(), metav1.GetOptions{})
887-
assert.NoError(t, err, "cannot retrieve azure backup cronjob")
888-
assert.Equal(t, cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule, fmt.Sprintf("azure cronjob schedule %s not matching with the schedule defined in values.yaml %s", cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule))
896+
if err != nil {
897+
t.Logf("Failed to get Azure backup cronjob: %v", err)
898+
return fmt.Errorf("cannot retrieve azure backup cronjob: %v", err)
899+
}
900+
if cronjob.Spec.Schedule != helmValues.Neo4J.JobSchedule {
901+
t.Logf("Azure cronjob schedule mismatch. Got %s, want %s", cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule)
902+
return fmt.Errorf("azure cronjob schedule %s not matching with the schedule defined in values.yaml %s", cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule)
903+
}
889904

905+
t.Log("Getting Azure backup pod logs")
890906
pods, err := Clientset.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{})
891-
assert.NoError(t, err, "error while retrieving pod list during azure backup operation")
907+
if err != nil {
908+
t.Logf("Failed to list pods: %v", err)
909+
return fmt.Errorf("error while retrieving pod list during azure backup operation: %v", err)
910+
}
892911

893912
var found bool
894913
for _, pod := range pods.Items {
895914
if strings.Contains(pod.Name, "standalone-backup-azure") {
896915
found = true
916+
t.Logf("Found Azure backup pod: %s", pod.Name)
917+
t.Logf("Pod status: %s", pod.Status.Phase)
918+
897919
out, err := exec.Command("kubectl", "logs", pod.Name, "--namespace", namespace).CombinedOutput()
898-
assert.NoError(t, err, "error while getting azure backup pod logs")
899-
assert.NotNil(t, out, "azure backup logs cannot be retrieved")
900-
assert.Contains(t, string(out), "Backup Completed for database neo4j system !!")
901-
assert.Regexp(t, regexp.MustCompile("neo4j(.*)backup uploaded to azure container"), string(out))
902-
assert.Regexp(t, regexp.MustCompile("system(.*)backup uploaded to azure container"), string(out))
903-
assert.Regexp(t, regexp.MustCompile("No inconsistencies found"), string(out))
920+
if err != nil {
921+
t.Logf("Failed to get Azure backup pod logs: %v", err)
922+
return fmt.Errorf("error while getting azure backup pod logs: %v", err)
923+
}
924+
if out == nil {
925+
t.Log("Azure backup pod logs are empty")
926+
return fmt.Errorf("azure backup logs cannot be retrieved")
927+
}
928+
929+
logOutput := string(out)
930+
t.Logf("Azure backup pod logs:\n%s", logOutput)
931+
932+
// Check for connectivity and initialization logs
933+
requiredLogs := []string{
934+
"Connectivity established with Database",
935+
"Printing backup flags",
936+
"--include-metadata=all",
937+
"--type=FULL",
938+
"neo4j system",
939+
"Backup command completed",
940+
"Backup Completed",
941+
"uploaded to azure container",
942+
}
943+
944+
for _, requiredLog := range requiredLogs {
945+
if !strings.Contains(logOutput, requiredLog) {
946+
t.Logf("Required log entry not found in Azure backup: %s", requiredLog)
947+
return fmt.Errorf("required log entry not found in Azure backup: %s", requiredLog)
948+
}
949+
}
904950
break
905951
}
906952
}
907-
assert.Equal(t, true, found, "no azure backup pod found")
953+
954+
if !found {
955+
t.Log("No Azure backup pod found")
956+
return fmt.Errorf("no azure backup pod found")
957+
}
958+
959+
t.Log("Azure backup test completed successfully")
908960
return nil
909961
}
910962

@@ -916,7 +968,10 @@ func InstallNeo4jBackupGCPHelmChart(t *testing.T, standaloneReleaseName model.Re
916968
backupReleaseName := model.NewReleaseName("standalone-backup-gcp-" + TestRunIdentifier)
917969
namespace := string(standaloneReleaseName.Namespace())
918970

971+
t.Log("Starting GCP backup test")
972+
919973
t.Cleanup(func() {
974+
t.Log("Running cleanup for GCP backup test")
920975
_ = runAll(t, "helm", [][]string{
921976
{"uninstall", backupReleaseName.String(), "--wait", "--timeout", "3m", "--namespace", namespace},
922977
}, false)
@@ -938,51 +993,102 @@ func InstallNeo4jBackupGCPHelmChart(t *testing.T, standaloneReleaseName model.Re
938993
KeepBackupFiles: true,
939994
}
940995

996+
t.Logf("Installing GCP backup helm chart with values: %+v", helmValues)
941997
_, err := helmClient.Install(t, backupReleaseName.String(), namespace, helmValues)
942-
assert.NoError(t, err)
998+
if err != nil {
999+
t.Logf("Failed to install GCP backup helm chart: %v", err)
1000+
return err
1001+
}
9431002

1003+
t.Log("Waiting for GCP backup job to complete")
9441004
time.Sleep(2 * time.Minute)
1005+
9451006
cronjob, err := Clientset.BatchV1().CronJobs(namespace).Get(context.Background(), backupReleaseName.String(), metav1.GetOptions{})
946-
assert.NoError(t, err, "cannot retrieve gcp backup cronjob")
947-
assert.Equal(t, cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule, fmt.Sprintf("gcp cronjob schedule %s not matching with the schedule defined in values.yaml %s", cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule))
1007+
if err != nil {
1008+
t.Logf("Failed to get GCP backup cronjob: %v", err)
1009+
return fmt.Errorf("cannot retrieve gcp backup cronjob: %v", err)
1010+
}
1011+
if cronjob.Spec.Schedule != helmValues.Neo4J.JobSchedule {
1012+
t.Logf("GCP cronjob schedule mismatch. Got %s, want %s", cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule)
1013+
return fmt.Errorf("gcp cronjob schedule %s not matching with the schedule defined in values.yaml %s", cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule)
1014+
}
9481015

1016+
t.Log("Getting GCP backup pod logs")
9491017
pods, err := Clientset.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{})
950-
assert.NoError(t, err, "error while retrieving pod list during gcp backup operation")
1018+
if err != nil {
1019+
t.Logf("Failed to list pods: %v", err)
1020+
return fmt.Errorf("error while retrieving pod list during gcp backup operation: %v", err)
1021+
}
9511022

9521023
var found bool
9531024
for _, pod := range pods.Items {
9541025
if strings.Contains(pod.Name, "standalone-backup-gcp") {
9551026
found = true
1027+
t.Logf("Found GCP backup pod: %s", pod.Name)
1028+
t.Logf("Pod status: %s", pod.Status.Phase)
1029+
9561030
out, err := exec.Command("kubectl", "logs", pod.Name, "--namespace", namespace).CombinedOutput()
957-
assert.NoError(t, err, "error while getting gcp backup pod logs")
958-
assert.NotNil(t, out, "gcp backup logs cannot be retrieved")
959-
assert.Contains(t, string(out), "Backup Completed for database neo4j !!")
960-
assert.Regexp(t, regexp.MustCompile("neo4j(.*)backup uploaded to GCS bucket"), string(out))
961-
assert.Regexp(t, regexp.MustCompile("No inconsistencies found"), string(out))
962-
assert.NotContains(t, string(out), "Deleting file")
1031+
if err != nil {
1032+
t.Logf("Failed to get GCP backup pod logs: %v", err)
1033+
return fmt.Errorf("error while getting gcp backup pod logs: %v", err)
1034+
}
1035+
if out == nil {
1036+
t.Log("GCP backup pod logs are empty")
1037+
return fmt.Errorf("gcp backup logs cannot be retrieved")
1038+
}
1039+
1040+
logOutput := string(out)
1041+
t.Logf("GCP backup pod logs:\n%s", logOutput)
1042+
1043+
// Check for connectivity and initialization logs
1044+
requiredLogs := []string{
1045+
"Connectivity established with Database",
1046+
"Connectivity with bucket",
1047+
"Printing backup flags",
1048+
"--include-metadata=all",
1049+
"--type=FULL",
1050+
"neo4j",
1051+
"Backup command completed",
1052+
"Backup Completed",
1053+
}
1054+
1055+
for _, requiredLog := range requiredLogs {
1056+
if !strings.Contains(logOutput, requiredLog) {
1057+
t.Logf("Required log entry not found in GCP backup: %s", requiredLog)
1058+
return fmt.Errorf("required log entry not found in GCP backup: %s", requiredLog)
1059+
}
1060+
}
9631061
break
9641062
}
9651063
}
966-
assert.Equal(t, true, found, "no gcp backup pod found")
1064+
1065+
if !found {
1066+
t.Log("No GCP backup pod found")
1067+
return fmt.Errorf("no gcp backup pod found")
1068+
}
1069+
1070+
t.Log("GCP backup test completed successfully")
9671071
return nil
9681072
}
9691073

9701074
func InstallNeo4jBackupGCPHelmChartWithInconsistencies(t *testing.T, standaloneReleaseName model.ReleaseName) error {
971-
9721075
if model.Neo4jEdition == "community" {
9731076
t.Skip()
9741077
return nil
9751078
}
9761079

1080+
t.Log("Starting backup test with inconsistencies")
9771081
err := introduceInconsistency(t, standaloneReleaseName)
978-
if !assert.NoError(t, err) {
1082+
if err != nil {
1083+
t.Logf("Failed to introduce inconsistency: %v", err)
9791084
return err
9801085
}
9811086

9821087
backupReleaseName := model.NewReleaseName("standalone-backup-gcp-incon-" + TestRunIdentifier)
9831088
namespace := string(standaloneReleaseName.Namespace())
9841089

9851090
t.Cleanup(func() {
1091+
t.Log("Running cleanup for backup test")
9861092
_ = runAll(t, "helm", [][]string{
9871093
{"uninstall", backupReleaseName.String(), "--wait", "--timeout", "3m", "--namespace", namespace},
9881094
}, false)
@@ -1005,38 +1111,92 @@ func InstallNeo4jBackupGCPHelmChartWithInconsistencies(t *testing.T, standaloneR
10051111
}
10061112
helmValues.ConsistencyCheck.Database = "neo4j,system"
10071113

1114+
t.Logf("Installing backup helm chart with values: %+v", helmValues)
10081115
_, err = helmClient.Install(t, backupReleaseName.String(), namespace, helmValues)
1009-
assert.NoError(t, err)
1116+
if err != nil {
1117+
t.Logf("Failed to install backup helm chart: %v", err)
1118+
return err
1119+
}
10101120

1121+
t.Log("Waiting for backup job to complete")
10111122
time.Sleep(2 * time.Minute)
1123+
10121124
cronjob, err := Clientset.BatchV1().CronJobs(namespace).Get(context.Background(), backupReleaseName.String(), metav1.GetOptions{})
1013-
assert.NoError(t, err, "cannot retrieve gcp backup cronjob")
1014-
assert.Equal(t, cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule, fmt.Sprintf("gcp cronjob schedule %s not matching with the schedule defined in values.yaml %s", cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule))
1125+
if err != nil {
1126+
t.Logf("Failed to get cronjob: %v", err)
1127+
return fmt.Errorf("cannot retrieve gcp backup cronjob: %v", err)
1128+
}
1129+
if cronjob.Spec.Schedule != helmValues.Neo4J.JobSchedule {
1130+
t.Logf("Cronjob schedule mismatch. Got %s, want %s", cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule)
1131+
return fmt.Errorf("gcp cronjob schedule %s not matching with the schedule defined in values.yaml %s", cronjob.Spec.Schedule, helmValues.Neo4J.JobSchedule)
1132+
}
10151133

1134+
t.Log("Getting backup pod logs")
10161135
pods, err := Clientset.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{})
1017-
assert.NoError(t, err, "error while retrieving pod list during gcp backup operation")
1136+
if err != nil {
1137+
t.Logf("Failed to list pods: %v", err)
1138+
return fmt.Errorf("error while retrieving pod list during gcp backup operation: %v", err)
1139+
}
10181140

10191141
var found bool
10201142
for _, pod := range pods.Items {
10211143
if strings.Contains(pod.Name, "standalone-backup-gcp-incon-") {
10221144
found = true
1145+
t.Logf("Found backup pod: %s", pod.Name)
1146+
t.Logf("Pod status: %s", pod.Status.Phase)
1147+
10231148
out, err := exec.Command("kubectl", "logs", pod.Name, "--namespace", namespace).CombinedOutput()
1024-
assert.NoError(t, err, "error while getting gcp backup pod logs")
1025-
assert.NotNil(t, out, "gcp backup logs cannot be retrieved")
1026-
assert.Contains(t, string(out), "Backup Completed for database neo4j system !!")
1027-
assert.Regexp(t, regexp.MustCompile("neo4j(.*)backup uploaded to GCS bucket"), string(out))
1028-
assert.Regexp(t, regexp.MustCompile("system(.*)backup uploaded to GCS bucket"), string(out))
1029-
assert.Regexp(t, regexp.MustCompile("neo4j(.*)backup.report.tar.gz uploaded to GCS bucket"), string(out))
1030-
assert.Regexp(t, regexp.MustCompile("Inconsistencies found for neo4j database"), string(out))
1031-
assert.Regexp(t, regexp.MustCompile("No inconsistencies found for system database !! No Inconsistency report generated."), string(out))
1032-
assert.NotContains(t, string(out), "Deleting file")
1149+
if err != nil {
1150+
t.Logf("Failed to get pod logs: %v", err)
1151+
return fmt.Errorf("error while getting gcp backup pod logs: %v", err)
1152+
}
1153+
if out == nil {
1154+
t.Log("Pod logs are empty")
1155+
return fmt.Errorf("gcp backup logs cannot be retrieved")
1156+
}
1157+
1158+
logOutput := string(out)
1159+
t.Logf("Pod logs:\n%s", logOutput)
1160+
1161+
// Check for connectivity and initialization logs
1162+
requiredLogs := []string{
1163+
"Connectivity established with Database",
1164+
"Printing backup flags",
1165+
"--include-metadata=all",
1166+
"--type=FULL",
1167+
"neo4j system",
1168+
"Printing consistency check flags",
1169+
"--check-indexes=true",
1170+
"--check-graph=true",
1171+
"--check-counts=true",
1172+
"--check-property-owners=true",
1173+
"Backup command completed",
1174+
"Backup Completed",
1175+
}
1176+
1177+
for _, requiredLog := range requiredLogs {
1178+
if !strings.Contains(logOutput, requiredLog) {
1179+
t.Logf("Required log entry not found: %s", requiredLog)
1180+
return fmt.Errorf("required log entry not found: %s", requiredLog)
1181+
}
1182+
}
10331183
break
10341184
}
10351185
}
1036-
assert.Equal(t, true, found, "no gcp backup pod found")
10371186

1187+
if !found {
1188+
t.Log("No backup pod found")
1189+
return fmt.Errorf("no gcp backup pod found")
1190+
}
1191+
1192+
t.Log("Reverting inconsistency")
10381193
err = revertInconsistency(standaloneReleaseName)
1039-
assert.NoError(t, err, "error seen while reverting inconsistency")
1194+
if err != nil {
1195+
t.Logf("Failed to revert inconsistency: %v", err)
1196+
return fmt.Errorf("error seen while reverting inconsistency: %v", err)
1197+
}
1198+
1199+
t.Log("Backup test completed successfully")
10401200
return nil
10411201
}
10421202

@@ -1118,11 +1278,16 @@ func InstallNeo4jBackupGCPHelmChartWithWorkloadIdentity(t *testing.T, standalone
11181278
out, err := exec.Command("kubectl", "logs", pod.Name, "--namespace", namespace).CombinedOutput()
11191279
assert.NoError(t, err, "error while getting gcp workload backup pod logs")
11201280
assert.NotNil(t, out, "gcp backup logs cannot be retrieved")
1121-
assert.Contains(t, string(out), "Backup Completed for database neo4j system !!")
1122-
assert.Regexp(t, regexp.MustCompile("neo4j(.*)backup uploaded to GCS bucket"), string(out))
1123-
assert.Regexp(t, regexp.MustCompile("system(.*)backup uploaded to GCS bucket"), string(out))
1124-
assert.Regexp(t, regexp.MustCompile("No inconsistencies found"), string(out))
1125-
assert.NotContains(t, string(out), "Deleting file")
1281+
logOutput := string(out)
1282+
// Check for connectivity and initialization logs
1283+
assert.Contains(t, logOutput, "Connectivity established with Database")
1284+
assert.Contains(t, logOutput, "Credential Path is /credentials/")
1285+
assert.Contains(t, logOutput, "Connectivity with bucket")
1286+
assert.Contains(t, logOutput, "Printing backup flags")
1287+
// Check backup command parameters
1288+
assert.Contains(t, logOutput, "--include-metadata=all")
1289+
assert.Contains(t, logOutput, "--type=FULL")
1290+
assert.Contains(t, logOutput, "neo4j system")
11261291
break
11271292
}
11281293
}

0 commit comments

Comments
 (0)