Merge branch 'master' of https://github.com/Percona-QA/percona-qa

Roel Van de Paar · Roel Van de Paar · commit 8cf70a724edc · 2018-09-04T23:22:06.000+10:00
diff --git a/pmm-tests/pgsql-specific-tests.bats b/pmm-tests/pgsql-specific-tests.bats
@@ -0,0 +1,125 @@
+#!/usr/bin/env bats
+
+## postgresql:metrics
+
+
+PGSQL_USER='psql'
+PGSQL_HOST='localhost'
+PGSQL_SOCK=5432
+
+@test "run pmm-admin under regular(non-root) user privileges" {
+if [[ $(id -u) -eq 0 ]] ; then
+	skip "Skipping this test, because you are running under root"
+fi
+run pmm-admin
+echo "$output"
+    [ "$status" -eq 1 ]
+    [ "${lines[0]}" = "pmm-admin requires superuser privileges to manage system services." ]
+}
+
+
+@test "run pmm-admin under root privileges" {
+if [[ $(id -u) -ne 0 ]] ; then
+	skip "Skipping this test, because you are NOT running under root"
+fi
+run pmm-admin
+echo "$output"
+    [ "$status" -eq 1 ]
+    [ "${lines[0]}" = "Usage:" ]
+}
+
+
+@test "run pmm-admin add postgresql:metrics based on running intsances" {
+	COUNTER=0
+	CURRENT_COUNT=$(sudo pmm-admin list | grep "postgresql:metrics" | sed 's|.*(||;s|)||' | wc -l)
+	for i in $(seq ${CURRENT_COUNT}); do
+		let COUNTER=COUNTER+1
+		run sudo pmm-admin add postgresql:metrics --port=${PGSQL_SOCK} --user=${PGSQL_USER} pgsql_metrics_$COUNTER
+		echo "$output"
+		[ "$status" -eq 0 ]
+		echo "${lines[0]}" | grep "OK, now monitoring"
+	done
+}
+
+@test "run pmm-admin add postgresql:metrics again based on running instances" {
+	COUNTER=0
+	CURRENT_COUNT=$(sudo pmm-admin list | grep "pgsql_metrics_" | grep -Eo '\/.*\)' | sed 's/)$//' | wc -l)
+	for i in $(seq ${CURRENT_COUNT}); do
+		let COUNTER=COUNTER+1
+		run sudo pmm-admin add postgresql:metrics --user=${PGSQL_USER} --port=${PGSQL_SOCK} pgsql_metrics_$COUNTER
+		echo "$output"
+			[ "$status" -eq 1 ]
+			[ "${lines[0]}" = "Error adding PostgreSQL metrics: there is already one instance with this name under monitoring." ]
+	done
+}
+
+
+@test "run pmm-admin remove postgresql:metrics" {
+	COUNTER=0
+	CURRENT_COUNT=$(sudo pmm-admin list | grep "pgsql_metrics_" | wc -l)
+	for i in $(seq ${CURRENT_COUNT}) ; do
+		let COUNTER=COUNTER+1
+		run sudo pmm-admin remove postgresql:metrics pgsql_metrics_$COUNTER
+		echo "$output"
+			[ "$status" -eq 0 ]
+			echo "${output}" | grep "OK, removed PostgreSQL metrics"
+	done
+}
+
+
+@test "run pmm-admin remove postgresql:metrics again" {
+	COUNTER=0
+	for i in $(sudo pmm-admin list | grep "pgsql_metrics_") ; do
+		let COUNTER=COUNTER+1
+		run sudo pmm-admin remove postgresql:metrics pgsql_metrics_$COUNTER
+		echo "$output"
+			[ "$status" -eq 0 ]
+			echo "${output}" | grep "no service found"
+	done
+}
+
+## add postgresql
+
+@test "run pmm-admin add postgresql" {
+	COUNTER=0
+	CURRENT_COUNT=$(sudo pmm-admin list | grep "postgresql:metrics" | sed 's|.*(||;s|).*||' | wc -l)
+	for i in $(seq ${CURRENT_COUNT}) ; do
+		let COUNTER=COUNTER+1
+		run sudo pmm-admin add postgresql --user=${PGSQL_USER} --port=${PGSQL_SOCK} postgresql_$COUNTER
+		echo "$output"
+			[ "$status" -eq 0 ]
+			echo "${lines[0]}" | grep "OK, already"
+			echo "${lines[1]}" | grep "OK, now"
+	done
+}
+
+@test "run pmm-admin add postgresql again" {
+	COUNTER=0
+	CURRENT_COUNT=$(sudo pmm-admin list | grep "postgresql:metrics" | grep "postgresql_" | sed 's|.*(||;s|).*||' | wc -l)
+	for i in $(seq ${CURRENT_COUNT}); do
+		let COUNTER=COUNTER+1
+		run sudo pmm-admin add postgresql --user=${PGSQL_USER} --port=${PGSQL_SOCK} postgresql_$COUNTER
+		echo "$output"
+			[ "$status" -eq 0 ]
+			echo "${lines[0]}" | grep "OK, already"
+			echo "${lines[1]}" | grep "OK, already"
+	done
+}
+
+@test "run pmm-admin remove postgresql" {
+	COUNTER=0
+	CURRENT_COUNT=$(sudo pmm-admin list | grep "postgresql:metrics" | grep "postgresql_" | sed 's|.*(||;s|).*||' | wc -l)
+	for i in $(seq ${CURRENT_COUNT}); do
+		let COUNTER=COUNTER+1
+		run sudo pmm-admin remove postgresql postgresql_$COUNTER
+		echo "$output"
+			[ "$status" -eq 0 ]
+			echo "${lines[0]}" | grep "OK, no system"
+			echo "${lines[1]}" | grep "OK, removed"
+	done
+
+}
+
+function teardown() {
+	echo "$output"
+}
diff --git a/pmm-tests/pmm-framework.sh b/pmm-tests/pmm-framework.sh
@@ -36,6 +36,7 @@ usage () {
   echo " --pmm-port                     Pass port for PMM docker"
   echo " --ps-version                   Pass Percona Server version info"
   echo " --ms-version                   Pass MySQL Server version info"
+  echo " --pgsql-version                Pass Postgre SQL server version Info"
   echo " --md-version                   Pass MariaDB Server version info"
   echo " --pxc-version                  Pass Percona XtraDB Cluster version info"
   echo " --mysqld-startup-options       Pass MySQL startup options. eg : --mysqld-startup-options='--innodb_buffer_pool_size=1G --innodb_log_file_size=1G'"
@@ -74,7 +75,7 @@ usage () {
 # Check if we have a functional getopt(1)
 if ! getopt --test
   then
-  go_out="$(getopt --options=u: --longoptions=addclient:,replcount:,pmm-server:,ami-image:,key-name:,ova-image:,ova-memory:,pmm-server-version:,pmm-port:,pmm-server-memory:,pmm-docker-memory:,pmm-server-username:,pmm-server-password:,query-source:,setup,with-replica,with-shrading,download,ps-version:,ms-version:,md-version:,pxc-version:,mysqld-startup-options:,mo-version:,mongo-with-rocksdb,add-docker-client,list,wipe-clients,wipe-docker-clients,wipe-server,disable-ssl,upgrade-server,upgrade-client,wipe,dev,with-proxysql,sysbench-data-load,sysbench-oltp-run,storage-engine:,compare-query-count,help \
+  go_out="$(getopt --options=u: --longoptions=addclient:,replcount:,pmm-server:,ami-image:,key-name:,ova-image:,ova-memory:,pmm-server-version:,pmm-port:,pmm-server-memory:,pmm-docker-memory:,pmm-server-username:,pmm-server-password:,query-source:,setup,with-replica,with-shrading,download,ps-version:,ms-version:,pgsql-version:,md-version:,pxc-version:,mysqld-startup-options:,mo-version:,mongo-with-rocksdb,add-docker-client,list,wipe-clients,wipe-docker-clients,wipe-server,disable-ssl,upgrade-server,upgrade-client,wipe,dev,with-proxysql,sysbench-data-load,sysbench-oltp-run,storage-engine:,compare-query-count,help \
   --name="$(basename "$0")" -- "$@")"
   test $? -eq 0 || exit 1
   eval set -- $go_out
@@ -159,6 +160,10 @@ do
     ms_version="$2"
     shift 2
     ;;
+    --pgsql-version )
+    pgsql_version="$2"
+    shift 2
+    ;;
     --md-version )
     md_version="$2"
     shift 2
@@ -369,13 +374,28 @@ sanity_check(){
   fi
 }
 
+sudo_check(){
+
+  USER=$1
+    # Sudo check
+  if [ "$(sudo -H -u ${USER} bash -c "echo 'test'" 2>/dev/null)" != "test" ]; then
+    echo "Error: sudo is not available or requires a password. This script needs to be able to use sudo, without password, from the userID that invokes it ($(whoami))"
+    echo "To get your setup correct, you may like to use a tool like visudo (use 'sudo visudo' or 'su' and then 'visudo') and consider adding the following line to the file:"
+    echo "$(whoami)   ALL=(ALL)      NOPASSWD:ALL"
+    echo "If you do not have sudo installed yet, try 'su' and then 'yum install sudo' or the apt-get equivalent"
+    echo "Terminating now."
+    exit 1
+  fi
+}
+
 if [[ -z "${ps_version}" ]]; then ps_version="5.7"; fi
 if [[ -z "${pxc_version}" ]]; then pxc_version="5.7"; fi
 if [[ -z "${ms_version}" ]]; then ms_version="8.0"; fi
 if [[ -z "${md_version}" ]]; then md_version="10.2"; fi
 if [[ -z "${mo_version}" ]]; then mo_version="3.4"; fi
 if [[ -z "${REPLCOUNT}" ]]; then REPLCOUNT="1"; fi
 if [[ -z "${ova_memory}" ]]; then ova_memory="2048";fi
+if [[ -z "${pgsql_version}" ]]; then pgsql_version="10.5";fi
 
 if [[ -z "$query_source" ]];then
   query_source=perfschema
@@ -650,7 +670,11 @@ get_basedir(){
         BASE_TAR=$(ls -1td $SERVER_STRING 2>/dev/null | grep ".tar" | head -n1)
         if [ ! -z $BASE_TAR ];then
           tar -xzf $BASE_TAR
-          BASEDIR=$(ls -1td $SERVER_STRING 2>/dev/null | grep -v ".tar" | head -n1)
+          if [[ "${PRODUCT_NAME}" == "postgresql" ]]; then
+            BASEDIR=$(ls -1td pgsql 2>/dev/null | grep -v ".tar" | head -n1)
+          else
+            BASEDIR=$(ls -1td $SERVER_STRING 2>/dev/null | grep -v ".tar" | head -n1)
+          fi
           BASEDIR="$WORKDIR/$BASEDIR"
           rm -rf $BASEDIR/node*
         else
@@ -670,7 +694,11 @@ get_basedir(){
       BASE_TAR=$(ls -1td $SERVER_STRING 2>/dev/null | grep ".tar" | head -n1)
       if [ ! -z $BASE_TAR ];then
         tar -xzf $BASE_TAR
-        BASEDIR=$(ls -1td $SERVER_STRING 2>/dev/null | grep -v ".tar" | head -n1)
+        if [[ "${PRODUCT_NAME}" == "postgresql" ]]; then
+            BASEDIR=$(ls -1td pgsql 2>/dev/null | grep -v ".tar" | head -n1)
+        else
+          BASEDIR=$(ls -1td $SERVER_STRING 2>/dev/null | grep -v ".tar" | head -n1)
+        fi
         BASEDIR="$WORKDIR/$BASEDIR"
         if [[ "${CLIENT_NAME}" == "mo" ]]; then
           sudo rm -rf $BASEDIR/data
@@ -782,6 +810,10 @@ add_clients(){
       NODE_NAME="MS_NODE"
       get_basedir mysql "mysql-${ms_version}*" "MySQL Server binary tar ball" ${ms_version}
       MYSQL_CONFIG="--init-file ${SCRIPT_PWD}/QRT_Plugin.sql --innodb_monitor_enable=all --performance_schema=ON"
+    elif [[ "${CLIENT_NAME}" == "pgsql" ]]; then
+      PORT_CHECK=501
+      NODE_NAME="PGSQL_NODE"
+      get_basedir postgresql "postgresql-${pgsql_version}*" "Postgre SQL Binary tar ball" ${pgsql_version}
     elif [[ "${CLIENT_NAME}" == "md" ]]; then
       PORT_CHECK=301
       NODE_NAME="MD_NODE"
@@ -795,8 +827,8 @@ add_clients(){
     elif [[ "${CLIENT_NAME}" == "mo" ]]; then
       get_basedir psmdb "percona-server-mongodb-${mo_version}*" "Percona Server Mongodb binary tar ball" ${mo_version}
     fi
-    if [[ "${CLIENT_NAME}" != "md"  && "${CLIENT_NAME}" != "mo" ]]; then
-    VERSION="$(${BASEDIR}/bin/mysqld --version | grep -oe '[58]\.[5670]' | head -n1)"
+    if [[ "${CLIENT_NAME}" != "md"  && "${CLIENT_NAME}" != "mo" && "${CLIENT_NAME}" != "pgsql" ]]; then
+      VERSION="$(${BASEDIR}/bin/mysqld --version | grep -oe '[58]\.[5670]' | head -n1)"
     if [ "$VERSION" == "5.7" -o "$VERSION" == "8.0" ]; then
         MID="${BASEDIR}/bin/mysqld   --default-authentication-plugin=mysql_native_password --initialize-insecure --basedir=${BASEDIR}"
       else
@@ -897,6 +929,39 @@ add_clients(){
           $BASEDIR/bin/mongo --quiet --eval "printjson(db.getSisterDB('admin').runCommand({addShard: 'r${k}/localhost:${PSMDB_PORTS[$n]}'}))"
         done
 	    fi
+    elif [[ "${CLIENT_NAME}" == "pgsql" ]]; then
+      echo "Creating postgresql Dedicated User psql"
+      IP_ADDRESS=$(ip route get 8.8.8.8 | awk -F"src " 'NR==1{split($2,a," ");print a[1]}')
+      if id psql >/dev/null 2>&1; then
+        echo "yes the user psql exists"
+      else
+        echo "No, the user psql does not exist, Adding"
+        sudo adduser --disabled-password --gecos "" psql
+      fi
+      PGSQL_PORT=5431
+      for j in `seq 1  ${ADDCLIENTS_COUNT}`;do
+        PGSQL_PORT=$((PGSQL_PORT+j))
+        cd ${BASEDIR}/bin
+        if [ -d ${BASEDIR}/${NODE_NAME}_${j}/data ]; then
+          echo "PGSQL Data Directory Exist, Removing old Directory, Stopping already running Server and creating a new one"
+          sudo -H -u psql bash -c "./pg_ctl -D ${BASEDIR}/${NODE_NAME}_${j}/data -l ${BASEDIR}/${NODE_NAME}_${j}/data/logfile -o '-F -p ${PGSQL_PORT}' stop" > /dev/null 2>&1;
+          sudo rm -r ${BASEDIR}/${NODE_NAME}_${j}
+          sudo mkdir -p ${BASEDIR}/${NODE_NAME}_${j}/data
+        else
+          sudo mkdir -p ${BASEDIR}/${NODE_NAME}_${j}/data
+        fi
+        sudo chown -R psql ${BASEDIR}/${NODE_NAME}_${j}/data
+        sudo_check psql
+        echo "Starting PGSQL server at port ${PGSQL_PORT}"
+        sudo -H -u psql bash -c "./pg_ctl -D ${BASEDIR}/${NODE_NAME}_${j}/data initdb" > /dev/null 2>&1;
+        sudo -H -u psql bash -c "./pg_ctl -D ${BASEDIR}/${NODE_NAME}_${j}/data -l ${BASEDIR}/${NODE_NAME}_${j}/data/logfile -o '-F -p ${PGSQL_PORT}' start" > /dev/null 2>&1;
+        if [ $disable_ssl -eq 1 ]; then
+          sudo pmm-admin add postgresql --user psql --host localhost --port ${PGSQL_PORT} --disable-ssl PGSQL-${NODE_NAME}-${j}
+          check_disable_ssl PGSQL-${NODE_NAME}-${j}
+        else
+          sudo pmm-admin add postgresql --user psql --host localhost --port ${PGSQL_PORT} PGSQL-${NODE_NAME}-${j}
+        fi
+      done
     else
       if [ -r ${BASEDIR}/lib/mysql/plugin/ha_tokudb.so ]; then
         TOKUDB_STARTUP="--plugin-load-add=tokudb=ha_tokudb.so --tokudb-check-jemalloc=0"
diff --git a/pmm-tests/pmm-testsuite.sh b/pmm-tests/pmm-testsuite.sh
@@ -70,6 +70,14 @@ function run_ps_specific_tests() {
   fi
 }
 
+function run_postgresql_specific_tests() {
+  if [[ $tap == 1 ]] ; then
+    bats --tap ${DIRNAME}/pgsql-specific-tests.bats
+  else
+    bats ${DIRNAME}/pgsql-specific-tests.bats
+  fi
+}
+
 function run_pxc_specific_tests() {
   if [[ $tap == 1 ]] ; then
     bats --tap ${DIRNAME}/pxc-specific-tests.bats
@@ -228,6 +236,11 @@ if [[ $instance_t == "ps" ]]; then
   run_ps_specific_tests
 fi
 
+if [[ $instance_t == "pgsql" ]]; then
+  echo "Running Postgre SQL specific tests"
+  run_postgresql_specific_tests
+fi
+
 
 if [[ $instance_t == "pxc" ]]; then
   echo "Running PXC specific tests"
diff --git a/pquery-clean-known.sh b/pquery-clean-known.sh
@@ -7,24 +7,21 @@
 SCRIPT_PWD=$(cd `dirname $0` && pwd)
 
 # Check if this an automated (pquery-reach.sh) run
+REACH=0  # Normal output
 if [ "$1" == "reach" ]; then
   REACH=1  # Minimal output, and no 2x enter required
-else
-  REACH=0  # Normal output
 fi
 
 # Check if this is a pxc run
+PXC=0
 if [ "$(grep 'PXC Mode:' ./pquery-run.log 2> /dev/null | sed 's|^.*PXC Mode[: \t]*||' )" == "TRUE" ]; then
-  export PXC=1
-else
-  export PXC=0
+  PXC=1
 fi
 
 # Check if this is a group replication run
+GRP_RPL=0
 if [ "$(grep 'Group Replication Mode:' ./pquery-run.log 2> /dev/null | sed 's|^.*Group Replication Mode[: \t]*||')" == "TRUE" ]; then
   GRP_RPL=1
-else
-  GRP_RPL=0
 fi
 
 # Current location checks
diff --git a/pquery-del-trial.sh b/pquery-del-trial.sh
@@ -7,6 +7,13 @@ if [[ $PXC -eq 1 ]];then
   PXC_NODE=$(echo $TRIAL | cut -d'-' -f2)
   TRIAL=$(echo $TRIAL | cut -d'-' -f1)
 fi
+
+# Check if this is a pxc run
+PXC=0
+if [ "$(grep 'PXC Mode:' ./pquery-run.log 2> /dev/null | sed 's|^.*PXC Mode[: \t]*||' )" == "TRUE" ]; then
+  PXC=1
+fi
+
 if [ "${TRIAL}" == "" ]; then
   echo "This script deletes a given pquery trial completely. Execute this script from within the pquery workdir"
   echo "Example: to delete trial 10 (./10), execute as: ./delete_single_trial.sh 10"
