Skip to content

Automated fix for refs/heads/prometheus-query #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: prometheus-query
Choose a base branch
from
Open

Automated fix for refs/heads/prometheus-query #9

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 34 additions & 32 deletions tools/run_tests/performance/prom.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.


# example usage: python3 prom.py --url=http://10.108.0.60:9090 --pod_type=driver --pod_type=client --container_name=main --container_name=sidecar --debug=true
import argparse
import json
import requests
import statistics
import argparse
from typing import Any, Dict, List

import requests


class Prom:
def __init__(
self,
url,
start,
end,
debug=False):

def __init__(self, url, start, end, debug=False):
self.url = url
self.start = start
self.end = end
Expand All @@ -51,7 +46,8 @@ def fetch_by_query(self, query: str) -> Any:

return resp.json()

def fetch_cpu_for_pod(self, container_matcher: str, pod_name: str) -> Dict[str, List[float]]:
def fetch_cpu_for_pod(self, container_matcher: str,
pod_name: str) -> Dict[str, List[float]]:
"""Fetch the cpu data for each pod and construct the container
name to cpu data list Dict"""
query = 'irate(container_cpu_usage_seconds_total{job="kubernetes-cadvisor",pod="' + pod_name + '",container=' + \
Expand All @@ -70,7 +66,8 @@ def fetch_cpu_for_pod(self, container_matcher: str, pod_name: str) -> Dict[str,

return cpu_container_name_to_data_list

def fetch_memory_for_pod(self, container_matcher: str, pod_name: str) -> Dict[str, List[float]]:
def fetch_memory_for_pod(self, container_matcher: str,
pod_name: str) -> Dict[str, List[float]]:
"""Fetch the memory data for each pod and construct the
container name to memory data list Dict"""
query = 'container_memory_usage_bytes{job="kubernetes-cadvisor",pod="' + pod_name + '",container=' + \
Expand All @@ -89,7 +86,9 @@ def fetch_memory_for_pod(self, container_matcher: str, pod_name: str) -> Dict[st

return memory_container_name_to_data_list

def fetch_cpu_and_memory_data(self, container_list: List[str], pod_list: List[str]) -> Dict[str, Dict[str, Dict[str, Dict[str, Any]]]]:
def fetch_cpu_and_memory_data(
self, container_list: List[str], pod_list: List[str]
) -> Dict[str, Dict[str, Dict[str, Dict[str, Any]]]]:
"""Fetch and process min, max, mean, std for the memory and cpu
data for each container in the container_list for each pod in
the pod_list and construct processed data group first by metric
Expand Down Expand Up @@ -132,8 +131,7 @@ def get_data_list_from_timeseries(data: Any) -> Dict[str, List[float]]:
"""Construct a Dict as keys are the container names and
values are a list of data taken from given timeserie data"""
if data['status'] != "success":
raise Exception("command failed: " +
data['status'] + str(data))
raise Exception("command failed: " + data['status'] + str(data))
if data['data']['resultType'] != "matrix":
raise Exception("resultType is not matrix: " +
data['data']['resultType'])
Expand All @@ -158,20 +156,26 @@ def compute_min_max_mean_std_for_each(data: List[float]) -> Dict[str, Any]:
max_value = max(data)
mean_value = statistics.mean(data)
std_value = statistics.pstdev(data)
processed_data = {"min": min_value, "max": max_value,
"mean": mean_value, "std": std_value}
processed_data = {
"min": min_value,
"max": max_value,
"mean": mean_value,
"std": std_value
}
return processed_data


def compute_min_max_mean_std(cpu_data_dicts: Dict[str, Dict[str, List[float]]]) -> Dict[str, Dict[str, Dict[str, Any]]]:
def compute_min_max_mean_std(
cpu_data_dicts: Dict[str, Dict[str, List[float]]]
) -> Dict[str, Dict[str, Dict[str, Any]]]:
"""Compute the min, max, mean and standard deviation for
given set of data"""
pod_name_to_data_dicts = {}
for pod_name, pod_data_dicts in cpu_data_dicts.items():
container_name_to_processed_data = {}
for container_name, data_list in pod_data_dicts.items():
container_name_to_processed_data[container_name] = compute_min_max_mean_std_for_each(
data_list)
container_name_to_processed_data[
container_name] = compute_min_max_mean_std_for_each(data_list)
pod_name_to_data_dicts[pod_name] = container_name_to_processed_data

return pod_name_to_data_dicts
Expand All @@ -181,9 +185,7 @@ def construct_pod_list(node_info_file: str, pod_types: List[str]) -> List[str]:
"""Construct a list of pod names to be queried"""
with open(node_info_file, "r") as f:
pod_names = json.load(f)
pod_type_to_name = {"client": [],
"driver": [],
"server": []}
pod_type_to_name = {"client": [], "driver": [], "server": []}

for client in pod_names["Clients"]:
pod_type_to_name["client"].append(client["Name"])
Expand All @@ -202,26 +204,24 @@ def construct_pod_list(node_info_file: str, pod_types: List[str]) -> List[str]:
def main() -> None:
argp = argparse.ArgumentParser(
description='Fetch cpu and memory stats from prometheus')
argp.add_argument('--url',
help='Prometheus base url',
required=True)
argp.add_argument('--url', help='Prometheus base url', required=True)
argp.add_argument("--scenario_result_file",
default='scenario_result.json',
type=str,
help="File contains epoch seconds for start and end time")
argp.add_argument("--node_info_file",
default='/var/data/qps_workers/node_info.json',
help="File contains pod name to query the metrics for")
argp.add_argument("--pod_type", action='append',
argp.add_argument("--pod_type",
action='append',
help="Pod_type to query the metrics for",
choices=['driver', 'client', 'server'],
required=True)
argp.add_argument("--container_name", action='append',
argp.add_argument("--container_name",
action='append',
help="The container names to query the metrics for",
required=True)
argp.add_argument('--debug',
default=False,
help='Print debug messages.')
argp.add_argument('--debug', default=False, help='Print debug messages.')
argp.add_argument("--export_file_name",
default='prometheus_query_result.json',
type=str,
Expand All @@ -231,8 +231,10 @@ def main() -> None:

with open(args.scenario_result_file, "r") as q:
scenario_result = json.load(q)
p = Prom(url=args.url, start=scenario_result["summary"]
["start"], end=scenario_result["summary"]["end"], debug=args.debug)
p = Prom(url=args.url,
start=scenario_result["summary"]["start"],
end=scenario_result["summary"]["end"],
debug=args.debug)

pod_list = construct_pod_list(args.node_info_file, args.pod_type)
processed_data = p.fetch_cpu_and_memory_data(
Expand Down