From 8385ef49a5b7a8ffba3ce7a2c8eb90c06c81a8e9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 19 Aug 2025 14:26:04 +0200 Subject: [PATCH] bin: Expand curl -k to --insecure for visibility - Should be okay from inside the cluster (harbor and opensearch scripts) - Should be okay from tests (tests/ and pipeline/) - No migrations changed --- bin/diagnostics.bash | 6 ++-- .../charts/harbor/init-harbor/files/init.sh | 14 ++++---- .../opensearch/configurer/files/configurer.sh | 32 +++++++++---------- pipeline/opensearch.bash | 2 +- pipeline/test/services/funcs.sh | 6 ++-- .../service-cluster/testOpensearch.sh | 30 ++++++++--------- tests/common/bats/harbor.bash | 2 +- 7 files changed, 47 insertions(+), 45 deletions(-) diff --git a/bin/diagnostics.bash b/bin/diagnostics.bash index 0ef2505b73..436d04c4b1 100755 --- a/bin/diagnostics.bash +++ b/bin/diagnostics.bash @@ -343,7 +343,7 @@ run_diagnostics_default_metrics() { query_and_parse() { query="${1}" print_func="${2}" - res="$(curl "${endpoint}/query_range" -k -s --header "${header}" --data-urlencode query="${query}" "${range_arg[@]}")" + res="$(curl "${endpoint}/query_range" --insecure -s --header "${header}" --data-urlencode query="${query}" "${range_arg[@]}")" if [[ $(jq '.data.result | length' <<<"${res}") -gt 0 ]]; then readarray metric_results_arr < <(jq -c '.data.result[]' <<<"${res}") for row in "${metric_results_arr[@]}"; do @@ -390,7 +390,7 @@ run_diagnostics_default_metrics() { # Opensearch status printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' - echo "Querying Opensearch cluster status" - res="$(curl "${endpoint}/query" -k -s --header "${header}" --data-urlencode query='elasticsearch_cluster_health_status{color=~"yellow|red"} > 0')" + res="$(curl "${endpoint}/query" --insecure -s --header "${header}" --data-urlencode query='elasticsearch_cluster_health_status{color=~"yellow|red"} > 0')" if [[ $(jq '.data.result | length' <<<"${res}") -gt 0 ]]; then echo "Opensearch is in $(jq '.data.result[0].metric.color' <<<"${res}") state!" fi @@ -403,7 +403,7 @@ run_diagnostics_query_metric() { endpoint="${domain}/api/v1/namespaces/thanos/services/thanos-query-query-frontend:9090/proxy/api/v1" header="Authorization: Bearer ${token}" - curl "${endpoint}/query" -k --header "${header}" --data-urlencode query="${1}" | jq + curl "${endpoint}/query" --insecure --header "${header}" --data-urlencode query="${1}" | jq } if [[ -z "${CK8S_PGP_FP:-}" ]]; then diff --git a/helmfile.d/charts/harbor/init-harbor/files/init.sh b/helmfile.d/charts/harbor/init-harbor/files/init.sh index 801679cad5..1a29722794 100644 --- a/helmfile.d/charts/harbor/init-harbor/files/init.sh +++ b/helmfile.d/charts/harbor/init-harbor/files/init.sh @@ -3,7 +3,7 @@ set -e validate_harbor() { echo "testing curl address ${ENDPOINT}" - exists=$(curl -k "${ENDPOINT}"/projects/1 | jq '.code') || { + exists=$(curl --insecure "${ENDPOINT}"/projects/1 | jq '.code') || { echo "ERROR L.${LINENO} - Harbor url ${ENDPOINT}/projects/1 cannot be reached." exit 1 } @@ -17,12 +17,12 @@ validate_harbor() { delete_library_project() { echo Removing project library from harbor # Curl will return status 500 even though it successfully removed the project. - curl -k -X DELETE -u admin:"${HARBOR_PASSWORD}" "${ENDPOINT}"/projects/1 >/dev/null + curl --insecure -X DELETE -u admin:"${HARBOR_PASSWORD}" "${ENDPOINT}"/projects/1 >/dev/null } create_new_private_default_project() { echo "Creating new private project default" - curl -k -X POST -u admin:"${HARBOR_PASSWORD}" "${ENDPOINT}"/projects --header 'Content-Type: application/json' --header 'Accept: application/json' --data '{ + curl --insecure -X POST -u admin:"${HARBOR_PASSWORD}" "${ENDPOINT}"/projects --header 'Content-Type: application/json' --header 'Accept: application/json' --data '{ "project_name": "default", "metadata": { "public": "0", @@ -41,7 +41,7 @@ init_harbor_state() { echo "Setting up initial harbor state" if [ "$exists" != "404" ]; then - name=$(curl -k -X GET "${ENDPOINT}"/projects/1 | jq '.name') + name=$(curl --insecure -X GET "${ENDPOINT}"/projects/1 | jq '.name') if [ "$name" = "\"library\"" ]; then delete_library_project @@ -54,7 +54,7 @@ init_harbor_state() { configure_OIDC() { echo "Configuring oidc support" - err=$(curl -k -X PUT "${ENDPOINT}/configurations" \ + err=$(curl --insecure -X PUT "${ENDPOINT}/configurations" \ -u admin:"${HARBOR_PASSWORD}" \ -H "accept: application/json" \ -H "Content-Type: application/json" \ @@ -79,7 +79,7 @@ configure_GC() { echo "Configuring GC" if [ "${GC_FORCE_CONFIGURE}" = "false" ]; then - res=$(curl -k -X GET -w "%{http_code}" "${ENDPOINT}/system/gc/schedule" \ + res=$(curl --insecure -X GET -w "%{http_code}" "${ENDPOINT}/system/gc/schedule" \ -u admin:"${HARBOR_PASSWORD}") # shellcheck disable=SC3057 @@ -96,7 +96,7 @@ configure_GC() { fi fi - err=$(curl -k -X PUT "${ENDPOINT}/system/gc/schedule" \ + err=$(curl --insecure -X PUT "${ENDPOINT}/system/gc/schedule" \ -u admin:"${HARBOR_PASSWORD}" \ -H "accept: application/json" \ -H "Content-Type: application/json" \ diff --git a/helmfile.d/charts/opensearch/configurer/files/configurer.sh b/helmfile.d/charts/opensearch/configurer/files/configurer.sh index 453b09903e..7300abdd89 100644 --- a/helmfile.d/charts/opensearch/configurer/files/configurer.sh +++ b/helmfile.d/charts/opensearch/configurer/files/configurer.sh @@ -50,7 +50,7 @@ wait_for_dashboards() { setup_dashboards() { echo echo "Setting up OpenSearch Dashboards" - resp=$(curl -s -kL -X POST "${osd_url}/api/saved_objects/_import?overwrite=true" \ + resp=$(curl -s --insecure -L -X POST "${osd_url}/api/saved_objects/_import?overwrite=true" \ -H "osd-xsrf: true" \ --form file=@/files/dashboards.ndjson -u "${auth}") success=$(echo "${resp}" | grep "^{" | jq -r '.success') @@ -84,7 +84,7 @@ register_s3_repository() { resp=$(curl --insecure -X PUT "${os_url}/_snapshot/${snapshot_repository}" \ -H 'Content-Type: application/json' \ -d' {"type": "s3", "settings":{ "bucket": "{{ .Values.config.s3.bucketName }}", "client": "default"}}' \ - -s -k -u "${auth}") + -s --insecure -u "${auth}") acknowledged=$(echo "${resp}" | grep "^{" | jq -r '.acknowledged') if [ "${acknowledged}" != "true" ]; then log_error_exit "Failed to register S3 repository" "${resp}" @@ -97,7 +97,7 @@ register_gcs_repository() { resp=$(curl --insecure -X PUT "${os_url}/_snapshot/${snapshot_repository}" \ -H 'Content-Type: application/json' \ -d' {"type": "gcs", "settings":{ "bucket": "{{ .Values.config.gcs.bucketName }}", "client": "default"}}' \ - -s -k -u "${auth}") + -s --insecure -u "${auth}") acknowledged=$(echo "${resp}" | grep "^{" | jq -r '.acknowledged') if [ "${acknowledged}" != "true" ]; then log_error_exit "Failed to register GSC repository" "${resp}" @@ -110,7 +110,7 @@ register_azure_repository() { resp=$(curl --insecure -X PUT "${os_url}/_snapshot/${snapshot_repository}" \ -H 'Content-Type: application/json' \ -d' {"type": "azure", "settings":{ "container": "{{ .Values.config.azure.containerName }}", "client": "default" }}' \ - -s -k -u "${auth}") + -s --insecure -u "${auth}") acknowledged=$(echo "${resp}" | grep "^{" | jq -r '.acknowledged') if [ "${acknowledged}" != "true" ]; then log_error_exit "Failed to register Azure repository" "${resp}" @@ -131,7 +131,7 @@ create_index_template() { echo "Creating index template from file '${filename}'" resp=$(curl --insecure -X PUT "${os_url}/_index_template/${name}?create=${strict}" \ -H "Content-Type: application/json" -s \ - -d@/files/${filename} -k -u "${auth}") + -d@/files/${filename} --insecure -u "${auth}") acknowledged=$(echo "${resp}" | grep "^{" | jq -r '.acknowledged') if [ "${acknowledged}" != "true" ]; then if [ "${overwrite_templates}" = "false" ] \ @@ -150,12 +150,12 @@ setup_policy() { update_policy() { policy=$1 policy_json=$(curl --insecure -X GET "${os_url}/_plugins/_ism/policies/${policy}" \ - -H "Content-Type: application/json" -k -s \ + -H "Content-Type: application/json" --insecure -s \ -u "${auth}") seq_no=$(echo "${policy_json}" | jq -r '._seq_no') primary_term=$(echo "${policy_json}" | jq -r '._primary_term') resp=$(curl --insecure -X PUT "${os_url}/_plugins/_ism/policies/${policy}?if_seq_no=${seq_no}&if_primary_term=${primary_term}" \ - -H "Content-Type: application/json" -k -s \ + -H "Content-Type: application/json" --insecure -s \ -d@"/files/${policy}.policy.json" \ -u "${auth}") id=$(echo "${resp}" | grep "^{" | jq -r '._id') @@ -169,7 +169,7 @@ setup_policy() { echo "Creating policy '${policy}'" resp=$(curl --insecure -X PUT "${os_url}/_plugins/_ism/policies/${policy}" \ -H "Content-Type: application/json" \ - -d@"/files/${policy}.policy.json" -k -s \ + -d@"/files/${policy}.policy.json" --insecure -s \ -u "${auth}") status=$(echo "${resp}" | grep "^{" | jq -r '.status') id=$(echo "${resp}" | grep "^{" | jq -r '._id') @@ -192,13 +192,13 @@ init_indices() { for idx in other kubernetes kubeaudit authlog; do indices=$(curl --insecure -X GET "${os_url}/_cat/aliases/${idx}" \ - -k -s -u "${auth}") + --insecure -s -u "${auth}") if echo "${indices}" | grep "true" > /dev/null; then # idx exists echo "Index '${idx}' already exists" else # create idx resp=$(curl --insecure -X PUT "${os_url}/%3C${idx}-default-%7Bnow%2Fd%7D-000001%3E" \ -H 'Content-Type: application/json' \ - -k -s -u "${auth}" \ + --insecure -s -u "${auth}" \ -d '{"aliases": {"'"${idx}"'": {"is_write_index": true }}}') acknowledged=$(echo "${resp}" | grep "^{" | jq -r '.acknowledged') if [ "${acknowledged}" = "true" ]; then @@ -214,7 +214,7 @@ create_role() { role_name="$1"; role_definition="$2" response=$(curl --insecure -X PUT "${os_url}/_plugins/_security/api/roles/${role_name}" \ -H 'Content-Type: application/json' \ - -k -s -u "${auth}" \ + --insecure -s -u "${auth}" \ -d "${role_definition}") status=$(echo "${response}" | grep "^{" | jq -r '.status') @@ -233,7 +233,7 @@ create_rolemapping() { rolemapping_name="$1"; role_definition="$2" response=$(curl --insecure -X PUT "${os_url}/_plugins/_security/api/rolesmapping/${rolemapping_name}" \ -H 'Content-Type: application/json' \ - -k -s -u "${auth}" \ + --insecure -s -u "${auth}" \ -d "${role_definition}") status=$(echo "${response}" | grep "^{" | jq -r '.status') @@ -252,7 +252,7 @@ create_user() { user_name="$1"; user_info="$2" response=$(curl --insecure -X PUT "${os_url}/_plugins/_security/api/internalusers/${user_name}" \ -H 'Content-Type: application/json' \ - -k -s -u "${auth}" \ + --insecure -s -u "${auth}" \ -d "${user_info}") status=$(echo "${response}" | grep "^{" | jq -r '.status') @@ -270,14 +270,14 @@ create_user() { create_update_snapshot_policy() { echo echo "Checking if snapshot policy exists" - policy_resp=$(curl --insecure -X GET "${os_url}/_plugins/_sm/policies/snapshot_management_policy" -s -k -u "${auth}") + policy_resp=$(curl --insecure -X GET "${os_url}/_plugins/_sm/policies/snapshot_management_policy" -s --insecure -u "${auth}") seq_no=$(echo "${policy_resp}" | grep "^{" | jq -r '._seq_no') primary_term=$(echo "${policy_resp}" | grep "^{" | jq -r '._primary_term') if [ "${seq_no}" != "null" ] && [ "${primary_term}" != "null" ]; then echo "Updating snapshot policy" resp=$(curl --insecure -X PUT "${os_url}/_plugins/_sm/policies/snapshot_management_policy?if_seq_no=${seq_no}&if_primary_term=${primary_term}" \ -H 'Content-Type: application/json' \ - -s -k -u "${auth}" \ + -s --insecure -u "${auth}" \ -d '{ "description": "Snapshot Management Policy", "creation": { @@ -315,7 +315,7 @@ create_update_snapshot_policy() { echo "Creating snapshot policy" resp=$(curl --insecure -X POST "${os_url}/_plugins/_sm/policies/snapshot_management_policy" \ -H 'Content-Type: application/json' \ - -s -k -u "${auth}" \ + -s --insecure -u "${auth}" \ -d '{ "description": "Snapshot Management Policy", "creation": { diff --git a/pipeline/opensearch.bash b/pipeline/opensearch.bash index bcc19b25dd..26da016f95 100755 --- a/pipeline/opensearch.bash +++ b/pipeline/opensearch.bash @@ -5,7 +5,7 @@ set -eu opensearch_url=https://opensearch.ops.pipeline-exoscale.elastisys.se/api/status retries=60 while [ ${retries} -gt 0 ]; do - result="$(curl --connect-timeout 20 --max-time 60 -ksIL -o /dev/null -w "%{http_code}" $opensearch_url || true)" + result="$(curl --connect-timeout 20 --max-time 60 --insecure -sIL -o /dev/null -w "%{http_code}" $opensearch_url || true)" [[ "${result}" == "401" ]] && echo "Opensearch is ready. Got status ${result}" break echo "Waiting for OpenSearch to be ready. Got status ${result}" diff --git a/pipeline/test/services/funcs.sh b/pipeline/test/services/funcs.sh index 98053318ca..3b3923e80b 100644 --- a/pipeline/test/services/funcs.sh +++ b/pipeline/test/services/funcs.sh @@ -259,7 +259,8 @@ function testEndpoint { args=( --connect-timeout 20 --max-time 60 - -ksIL + --insecure + -sIL -o /dev/null -X GET -w "%{http_code}" @@ -294,7 +295,8 @@ function testEndpointProtected { args=( --connect-timeout 20 --max-time 60 - -ksI + --insecure + -sI -o /dev/null -X GET -w "%{http_code}" diff --git a/pipeline/test/services/service-cluster/testOpensearch.sh b/pipeline/test/services/service-cluster/testOpensearch.sh index 7ea155190b..84292dff79 100644 --- a/pipeline/test/services/service-cluster/testOpensearch.sh +++ b/pipeline/test/services/service-cluster/testOpensearch.sh @@ -84,7 +84,7 @@ function sc_opensearch_checks() { check_opensearch_cluster_health() { echo -ne "Checking if opensearch cluster is healthy ... " - cluster_health=$(curl -sk -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_cluster/health") + cluster_health=$(curl -s --insecure -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_cluster/health") status=$(echo "$cluster_health" | jq -r '.status') if [[ $status != "green" ]]; then echo -e "failure ❌" @@ -99,9 +99,9 @@ check_opensearch_snapshots_status() { no_error=true debug_msg="" repo_name=$(yq -e '.opensearch.snapshot.repository' "${config['config_file_sc']}") - repo_exists_status=$(curl -sk -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_snapshot/${repo_name}" | jq "select(.error)") + repo_exists_status=$(curl -s --insecure -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_snapshot/${repo_name}" | jq "select(.error)") if [[ -z "$repo_exists_status" ]]; then - snapshots=$(curl -sk -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_cat/snapshots/${repo_name}") + snapshots=$(curl -s --insecure -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_cat/snapshots/${repo_name}") error=$(echo "$snapshots" | jq '.error' 2>/dev/null || true) failed=$(echo "$snapshots" | grep 'FAILED' || true) partial=$(echo "$snapshots" | grep 'PARTIAL' || true) @@ -159,7 +159,7 @@ check_opensearch_indices() { no_error=true for index in 'other' 'kubernetes' 'kubeaudit' 'authlog'; do - res=$(curl -w "%{http_code}" -o /dev/null -ksIL -u admin:"${adminPassword}" -X HEAD "https://opensearch.${opsDomain}/${index}") + res=$(curl -w "%{http_code}" -o /dev/null --insecure -sIL -u admin:"${adminPassword}" -X HEAD "https://opensearch.${opsDomain}/${index}") if [[ $res != "200" ]]; then debug_msg+="[ERROR] Missing index : ${index}\n" no_error=false @@ -177,7 +177,7 @@ check_opensearch_indices() { check_opensearch_breakers() { echo -ne "Checking opensearch breakers ... " - breakers_data=$(curl -sk -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_nodes/_all/stats/breaker") + breakers_data=$(curl -s --insecure -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_nodes/_all/stats/breaker") no_error=true debug_msg="" nodes_data=$(echo "$breakers_data" | jq ".nodes") @@ -211,7 +211,7 @@ check_opensearch_aliases() { no_error=true debug_msg="" - curl -sk -o /tmp/response -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_cat/aliases" + curl -s --insecure -o /tmp/response -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_cat/aliases" aliases=$(awk '{print $1}'