feat(analytics): Graph-Analytics-Benchmarks

analytical_engine/benchmarks/framework/.env.example

@@ -0,0 +1 @@
+OPENAI_API_KEY="your_openai_api_key_here"

analytical_engine/benchmarks/framework/config/flash-mpijob-template.yaml

@@ -0,0 +1,83 @@
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+  name: flash-mpijob
+spec:
+  slotsPerWorker: ${SLOTS_PER_WORKER}
+  runPolicy:
+    cleanPodPolicy: None
+    ttlSecondsAfterFinished: 60
+  sshAuthMountPath: /home/mpiuser/.ssh
+  mpiReplicaSpecs:
+    Launcher:
+      replicas: 1
+      template:
+        spec:
+          ${VOLUMES_BLOCK}
+          containers:
+          - image: graphanalysisbenchmarks/flash-mpi:v0.1
+            name: mpi-launcher
+            securityContext:
+              runAsUser: 1000
+            ${VOLUME_MOUNTS_BLOCK}
+            command: ["/bin/bash", "-c"]
+            args:
+              - |
+                set -e
+                echo "[INFO] Waiting for DNS sync..."; sleep 5;
+
+                if [ "${ALGORITHM}" = "sssp" ]; then
+
+                  if [ "${SINGLE_MACHINE}" = "1" ]; then
+                    echo "Running in single-machine mode..."
+                    mpirun -n ${MPIRUN_NP} ./flash format /opt/data_sssp/ /opt/scratch/gfs/ data_sssp
+                    echo "Running ${ALGORITHM} in single-machine mode..."
+                    mpirun -n ${MPIRUN_NP} ./${ALGORITHM} /opt/scratch/gfs/ data_sssp ${ALGORITHM_PARAMETER}
+
+                  else
+                    echo "Running in multi-machine mode..."
+                    mpirun -n ${MPIRUN_NP} -hostfile /etc/mpi/hostfile ./flash format /opt/data_sssp/ /opt/scratch/gfs/ data_sssp
+                    mpirun -n ${MPIRUN_NP} -hostfile /etc/mpi/hostfile ./${ALGORITHM} /opt/scratch/gfs/ data_sssp ${ALGORITHM_PARAMETER}
+
+                  fi
+
+                else
+                  if [ "${SINGLE_MACHINE}" = "1" ]; then
+                    echo "Running in single-machine mode..."
+                    mpirun -n ${MPIRUN_NP} ./flash format /opt/data/ /opt/scratch/gfs/ data
+                    echo "Running ${ALGORITHM} in single-machine mode..."
+                    mpirun -n ${MPIRUN_NP} ./${ALGORITHM} /opt/scratch/gfs/ data ${ALGORITHM_PARAMETER}
+
+                  else
+                    echo "Running in multi-machine mode..."
+                    mpirun -n ${MPIRUN_NP} -hostfile /etc/mpi/hostfile ./flash format /opt/data/ /opt/scratch/gfs/ data
+                    mpirun -n ${MPIRUN_NP} -hostfile /etc/mpi/hostfile ./${ALGORITHM} /opt/scratch/gfs/ data ${ALGORITHM_PARAMETER}
+
+                  fi
+                fi
+
+            resources:
+              limits:
+                cpu: ${CPU}
+                memory: ${MEMORY}
+    Worker:
+      replicas: ${REPLICAS}
+      template:
+        spec:
+          ${VOLUMES_BLOCK}
+          containers:
+          - image: graphanalysisbenchmarks/flash-mpi:v0.1
+            name: mpi-worker
+            securityContext:
+              runAsUser: 1000
+            ${VOLUME_MOUNTS_BLOCK}
+            command:
+            - /usr/sbin/sshd
+            args:
+            - -De
+            - -f
+            - /home/mpiuser/.sshd_config
+            resources:
+              limits:
+                cpu: ${CPU}
+                memory: ${MEMORY}

analytical_engine/benchmarks/framework/config/grape-mpijob-template.yaml

@@ -0,0 +1,69 @@
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+  name: grape-mpijob
+spec:
+  slotsPerWorker: ${SLOTS_PER_WORKER}
+  runPolicy:
+    cleanPodPolicy: None
+    ttlSecondsAfterFinished: 60
+  sshAuthMountPath: /home/mpiuser/.ssh
+  mpiReplicaSpecs:
+    Launcher:
+      replicas: 1
+      template:
+        spec:
+          ${VOLUMES_BLOCK}
+          containers:
+          - image: graphanalysisbenchmarks/grape-mpi:v0.1
+            name: mpi-launcher
+            securityContext:
+              runAsUser: 1000
+            ${VOLUME_MOUNTS_BLOCK}
+            command: ["/bin/bash", "-c"]
+            args:
+              - |
+                set -e
+                echo "[INFO] Waiting for DNS sync..."; sleep 10;
+
+                if [ "${SINGLE_MACHINE}" = "1" ]; then
+                  echo "Running in single-machine mode..."
+                  if [ "${ALGORITHM}" = "sssp" ]; then
+                    time mpirun -n 1 ./run_app --app_concurrency ${MPIRUN_NP} --vfile /opt/data/graph_sssp.v --efile /opt/data/graph_sssp.e --application ${ALGORITHM} --bfs_source 0 --sssp_source 0 --pr_d 0.85 --pr_mr 10 --cdlp_mr 10 --opt 
+                  else
+                    time mpirun -n 1 ./run_app --app_concurrency ${MPIRUN_NP} --vfile /opt/data/graph.v --efile /opt/data/graph.e --application ${ALGORITHM} --bfs_source 0 --sssp_source 0 --pr_d 0.85 --pr_mr 10 --cdlp_mr 10 --opt 
+                  fi
+                else
+                  echo "Running in multi-machine mode..."
+                  if [ "${ALGORITHM}" = "sssp" ]; then
+                    time mpirun -n ${REPLICAS} --hostfile /etc/mpi/hostfile ./run_app --app_concurrency 32 --vfile /opt/data/graph_sssp.v --efile /opt/data/graph_sssp.e --application ${ALGORITHM} --bfs_source 0 --sssp_source 0 --pr_d 0.85 --pr_mr 10 --cdlp_mr 10 --opt 
+                  else
+                    time mpirun -n ${REPLICAS} --hostfile /etc/mpi/hostfile ./run_app --app_concurrency 32 --vfile /opt/data/graph.v --efile /opt/data/graph.e --application ${ALGORITHM} --bfs_source 0 --sssp_source 0 --pr_d 0.85 --pr_mr 10 --cdlp_mr 10 --opt 
+                  fi
+                fi
+
+            resources:
+              limits:
+                cpu: ${CPU}
+                memory: ${MEMORY}
+    Worker:
+      replicas: ${REPLICAS}
+      template:
+        spec:
+          ${VOLUMES_BLOCK}
+          containers:
+          - image: graphanalysisbenchmarks/grape-mpi:v0.1
+            name: mpi-worker
+            securityContext:
+              runAsUser: 1000
+            ${VOLUME_MOUNTS_BLOCK}
+            command:
+            - /usr/sbin/sshd
+            args:
+            - -De
+            - -f
+            - /home/mpiuser/.sshd_config
+            resources:
+              limits:
+                cpu: ${CPU}
+                memory: ${MEMORY}

analytical_engine/benchmarks/framework/config/gthinker-mpijob-template.yaml

@@ -0,0 +1,168 @@
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+  name: gthinker-mpijob
+spec:
+  slotsPerWorker: ${SLOTS_PER_WORKER}          
+  runPolicy:
+    cleanPodPolicy: None
+    ttlSecondsAfterFinished: 120
+  sshAuthMountPath: /home/mpiuser/.ssh
+
+  mpiReplicaSpecs:
+    Launcher:
+      replicas: 1
+      template:
+        spec:
+          restartPolicy: OnFailure
+          volumes:
+            - name: hadoop-config
+              configMap:
+                name: my-hadoop-cluster-hadoop   
+            - name: scratch
+              emptyDir: {}
+            ${VOLUMES_BLOCK}
+            - name: shared-mpi-data
+              emptyDir: {}
+
+
+          containers:
+            - name: mpi-launcher
+              image: graphanalysisbenchmarks/gthinker-mpi:v0.1      
+              imagePullPolicy: IfNotPresent
+              securityContext:
+                runAsUser: 1000
+              env:
+                - name: HADOOP_CONF_DIR
+                  value: "/etc/hadoop"
+
+                - name: HDFS_NN_SERVICE
+                  value: my-hadoop-cluster-hadoop-hdfs-nn
+                - name: HDFS_NN_PORT
+                  value: "9000"                   
+
+                - name: CLEAN_OUTPUT
+                  value: "1"
+
+                - name: JAVA_HOME
+                  value: /usr/lib/jvm/java-17-openjdk-amd64
+                - name: PATH
+                  value: "$(JAVA_HOME)/bin:/usr/local/hadoop/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+
+              volumeMounts:
+                - name: hadoop-config
+                  mountPath: /etc/hadoop
+                - name: scratch
+                  mountPath: /opt/scratch
+                ${VOLUME_MOUNTS_BLOCK}
+                - name: shared-mpi-data
+                  mountPath: /mnt/mpi
+
+              command: ["/bin/bash"]
+              args:
+                - "-c"
+                - |
+                    # 开启调试模式，如果任何命令失败则立即退出
+                    # set -euxo pipefail # <- 修正了拼写
+
+                    echo "[INFO] Waiting 10s for network/DNS to fully ready..."
+                    sleep 10
+
+                    # 1. 准备 HDFS 数据
+                    #    这里的 graph.txt 将由 envsubst 替换
+                    echo "[INFO] Preparing HDFS data for dataset: graph.txt"
+                    DATASET_HDFS_PATH="/user/mpiuser/graph.txt"
+
+                    # hdfs dfs -rm -r -f -skipTrash "${DATASET_HDFS_PATH}" || true
+
+                    if hdfs dfs -test -e "${DATASET_HDFS_PATH}"; then
+                       echo "[INFO] Dataset already exists on HDFS."
+                    else
+
+                        echo "[INFO] Dataset not found on HDFS, uploading..."
+                        HADOOP_USER_NAME=root hdfs dfs -mkdir -p /user/mpiuser
+                        HADOOP_USER_NAME=root hdfs dfs -chown mpiuser:supergroup /user/mpiuser
+                        HADOOP_USER_NAME=root hdfs dfs -chmod 775 /user/mpiuser
+                        hdfs dfs -put "/opt/data/graph.txt" "${DATASET_HDFS_PATH}"
+
+                    fi
+                    echo "[INFO] HDFS data is ready."
+
+                    hdfs dfs -ls /user/mpiuser/
+
+                    cat /etc/hadoop/core-site.xml
+
+                    HADOOP_CP=$(cat /etc/hadoop_classpath)
+
+                    CONVERTED_HOSTFILE="/mnt/mpi/hostfile" 
+                    sed 's/ slots=/:/' /etc/mpi/hostfile > "${CONVERTED_HOSTFILE}"
+
+                    time mpiexec \
+                      -np ${REPLICAS} \
+                      --hostfile "${CONVERTED_HOSTFILE}" \
+                      -verbose \
+                      -env CLASSPATH "${HADOOP_CP}" \
+                      -env LD_LIBRARY_PATH "/usr/lib/jvm/java-17-openjdk-amd64/lib/server:/usr/local/hadoop/lib/native:/usr/local/bin:/usr/local/lib:/usr/local/openmpi/bin:/usr/local/openmpi/lib" \
+                      -env JAVA_HOME "/usr/lib/jvm/java-17-openjdk-amd64" \
+                      -env HADOOP_CONF_DIR "/etc/hadoop" \
+                      /opt/gthinker/${ALGORITHM}/run "${DATASET_HDFS_PATH}" ${SLOTS_PER_WORKER}
+
+              resources:
+                limits:
+                  cpu: ${CPU}
+                  memory: ${MEMORY}
+                requests:
+                  cpu: ${CPU}
+                  memory: ${MEMORY}
+
+    Worker:
+      replicas: ${REPLICAS}
+      template:
+        spec:
+          restartPolicy: OnFailure
+          volumes:
+            - name: hadoop-config
+              configMap:
+                name: my-hadoop-cluster-hadoop
+            - name: scratch
+              emptyDir: {}
+            ${VOLUMES_BLOCK}
+            - name: shared-mpi-data
+              emptyDir: {}
+          containers:
+            - name: mpi-worker
+              image: graphanalysisbenchmarks/gthinker-mpi:v0.1
+              imagePullPolicy: IfNotPresent
+              securityContext:
+                runAsUser: 1000
+              env:
+                - name: JAVA_HOME
+                  value: /usr/lib/jvm/java-17-openjdk-amd64
+                - name: PATH
+                  value: "$(JAVA_HOME)/bin:/usr/local/hadoop/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+                - name: HADOOP_CONF_DIR
+                  value: "/etc/hadoop"
+                - name: HDFS_NN_SERVICE
+                  value: my-hadoop-cluster-hadoop-hdfs-nn
+                - name: HDFS_NN_PORT
+                  value: "9000"  
+
+
+              volumeMounts:
+                - name: hadoop-config
+                  mountPath: /etc/hadoop
+                - name: scratch
+                  mountPath: /opt/scratch
+                ${VOLUME_MOUNTS_BLOCK}
+                - name: shared-mpi-data
+                  mountPath: /mnt/mpi
+              command: ["/usr/sbin/sshd"]
+              args: ["-De","-f","/home/mpiuser/.sshd_config"]
+              resources:
+                limits:
+                  cpu: ${CPU}
+                  memory: ${MEMORY}
+                requests:
+                  cpu: ${CPU}
+                  memory: ${MEMORY}

analytical_engine/benchmarks/framework/config/ligra-mpijob-template.yaml

@@ -0,0 +1,58 @@
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+  name: ligra-mpijob
+spec:
+  slotsPerWorker: ${SLOTS_PER_WORKER}
+  runPolicy:
+    cleanPodPolicy: None
+    ttlSecondsAfterFinished: 60
+  sshAuthMountPath: /home/mpiuser/.ssh
+  mpiReplicaSpecs:
+    Launcher:
+      replicas: 1
+      template:
+        spec:
+          ${VOLUMES_BLOCK}
+          containers:
+          - image: graphanalysisbenchmarks/ligra-mpi:v0.1
+            name: mpi-launcher
+            securityContext:
+              runAsUser: 1000
+            ${VOLUME_MOUNTS_BLOCK}
+            command: ["/bin/bash", "-c"]
+            args:
+              - |
+                set -e
+                echo "[INFO] Waiting for DNS sync..."; sleep 5;
+                if [ "${ALGORITHM}" = "BellmanFord" ]; then
+                  time ./${ALGORITHM}${MPIRUN_NP} -rounds 1 /opt/data/graph_sssp.txt
+                else
+                  time ./${ALGORITHM}${MPIRUN_NP} -rounds 1 /opt/data/graph.txt
+                fi
+
+            resources:
+              limits:
+                cpu: ${CPU}
+                memory: ${MEMORY}
+    Worker:
+      replicas: ${REPLICAS}
+      template:
+        spec:
+          ${VOLUMES_BLOCK}
+          containers:
+          - image: graphanalysisbenchmarks/ligra-mpi:v0.1
+            name: mpi-worker
+            securityContext:
+              runAsUser: 1000
+            ${VOLUME_MOUNTS_BLOCK}
+            command:
+            - /usr/sbin/sshd
+            args:
+            - -De
+            - -f
+            - /home/mpiuser/.sshd_config
+            resources:
+              limits:
+                cpu: ${CPU}
+                memory: ${MEMORY}