diff --git a/Makefile.environment b/Makefile.environment new file mode 100644 index 0000000000..91195b2b7b --- /dev/null +++ b/Makefile.environment @@ -0,0 +1,47 @@ +HIVE_ROOT := $(shell git rev-parse --show-toplevel) +export PATH := $(HIVE_ROOT)/.tmp/_output/bin:$(PATH) +export KUBECONFIG := $(HIVE_ROOT)/.tmp/_output/dev-hive.kubeconfig + +all: install_environment_bins create_cluster_registry build_push_image deploy + +# Install dependency bins, make hive & run containerd, buildkitd +install_environment_bins: + @echo "Installing go and additional dependencies" + ./hack/install_dev_env.sh + +# Create Kind Cluster & registry +create_cluster_registry: + @echo "Creating a Kind cluster" + ./hack/create_cluster_registry.sh + +# Build and push image +build_push_image: + @echo "Building and pushing the image" + ./hack/buildkitd_build_push_image.sh + +# Deploy & install certs +deploy: + set -x + @echo "Deploying the application" + ./hack/deploy_dev_hive.sh + HIVE_NS=dev-hive ./hack/hiveadmission-dev-cert.sh + +# Scale down operator +scale-down-operator: + oc scale -n dev-hive deployment.v1.apps/hive-operator --replicas=0 + +# Scale down controllers +scale-down-controllers: + oc scale -n dev-hive deployment.v1.apps/hive-controllers --replicas=0 + +# Run operator +run-operator: scale-down-operator + ./bin/operator --log-level=debug + +# Run controller +run-controller: scale-down-controllers + ./bin/manager --log-level=debug + +# Clean the environment +prune-dev-env: + ./hack/prune_dev_env.sh diff --git a/docs/developing.md b/docs/developing.md index ae51f26ad9..fe20383c61 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -15,6 +15,8 @@ - [Running Code Locally](#running-code-locally) - [hive-operator](#hive-operator) - [hive-controllers](#hive-controllers) +- [Rootless development environment setup](#rootless-development-environment-setup) + - [Setting up VScode for hive development](#setting-up-vscode-for-hive-development) - [Developing Hiveutil Install Manager](#developing-hiveutil-install-manager) - [Enable Debug Logging In Hive Controllers](#enable-debug-logging-in-hive-controllers) - [Using Serving Certificates](#using-serving-certificates) @@ -166,7 +168,7 @@ do the trick. ```shell export EL8_BUILD_IMAGE=registry.ci.openshift.org/openshift/release:golang-1.23 export EL9_BUILD_IMAGE=registry.ci.openshift.org/openshift/release:golang-1.23 -export BASE_IMAGE=registry.ci.openshift.org/origin/4.16:base +export BASE_IMAGE=quay.io/centos/centos:stream9 # NOTE: This produces images which are not FIPS-compliant. export GO="CGO_ENABLED=0 go" ``` @@ -208,6 +210,77 @@ HIVE_NS="hive" make run Kind users should also specify `HIVE_IMAGE="localhost:5000/hive:latest"` as the default image location cannot be authenticated to from Kind clusters, resulting in inability to launch install pods. +# Rootless development environment setup +An automated script can set up your development environment - download go, prerequisite binaries, create a cluster, registry, and deploy hive. The script downloads binaries, stores them in the repository's root, and calls them from there. The setup is OS-independent and runs rootless, however, it currently only works on amd64 machines. + +To prepare the environment, run + +```bash +make -f Makefile.environment +``` +Containerd/nerdctl is used for container management, image is built using buildkitd, cluster management is done with kubectl and oc. + +Hive will be deployed into a cluster with a *'dev-hive'* namespace. + +After the setup is finished, to manage the cluster and do other actions from your shell, source */hack/dev_env_vars.sh* first in order to get access to the necessary paths and environment variables. + +```bash +source hack/dev_env_vars.sh +``` + +To clean up your environment *(remove containers, build cache, /.tmp, /.kube, /hiveadmission-certs, background processes)*, please run + +```bash +make -f Makefile.environment prune-dev-env +``` + +### Setting up VScode for hive development +Prerequisites +- [VScode](https://code.visualstudio.com/docs/setup/linux) +- VScode Go extension *(Ctrl+Shift+X and search for Go)* +- [Delve](https://github.com/go-delve/delve/tree/master/Documentation/installation) +- *source hack/dev_env_vars.sh* + +After opening Hive in VScode, you can seup the IDE for debugging by pressing *ctrl + shift + D* and clicking on *create launch.json*. + +The launch.json will govern which aspect to debug, e.g. the controller or the operator. Pressing F5 starts the debug process. + +Prior to debugging either the controller, or the operator, they need to be scaled down first. This can be done by running +```bash +make -f Makefile.environment scale-down-operator +# or +make -f Makefile.environment scale-down-controllers +``` + +Example of launch.json for debugging the controller: + +*Note: Metrics are a necessary part of the controller, however, for development purposes they are not needed and we can go around it creating an empty ({}) metrics.json file. The .json file is created automatically during the deploy task, exported here as METRICS_CONFIG_FILE* + +```bash +{ + "version": "0.2.0", + "configurations": [ + { + "name": "HiveController", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/manager/main.go", + "env": { + "HIVE_NS": "dev-hive", + "KUBECONFIG":"${workspaceFolder}/.tmp/_output/dev-hive.kubeconfig", + "HIVE_SKIP_LEADER_ELECTION":"1", + "METRICS_CONFIG_FILE":"${workspaceFolder}/.tmp/_output/metrics.json", + "HIVE_MACHINEPOOL_POD_NAME":"hive-machinepool-0", + "HIVE_OPERATOR_NS":"dev-hive", + "HIVE_CLUSTERSYNC_POD_NAME":"hive-clustersync-0", + }, + "args": ["--log-level=debug"] + }, + ] + } +``` + ## Developing Hiveutil Install Manager We use a hiveutil subcommand for the install-manager, in pods and thus in an image to wrap the openshift-install process and upload artifacts to Hive. Developing this is tricky because it requires a published image and ClusterImageSet. Instead, you can hack together an environment as follows: diff --git a/hack/buildkitd_build_push_image.sh b/hack/buildkitd_build_push_image.sh new file mode 100755 index 0000000000..d8b289115e --- /dev/null +++ b/hack/buildkitd_build_push_image.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +HIVE_ROOT="$(git rev-parse --show-toplevel)" +CNI_PATH="$HIVE_ROOT"/.tmp/_output/cni/bin +export HIVE_ROOT +export CNI_PATH +export PATH=$HIVE_ROOT/.tmp/_output/bin:$PATH + +touch "$HIVE_ROOT/.tmp/_output/config.json" + +# build and push image to localhost:5000/hive:latest +buildctl --addr unix:///run/user/$UID/buildkit/buildkitd.sock build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=. \ + --secret id=docker,src="$HIVE_ROOT/.tmp/_output/config.json" \ + --opt build-arg:EL8_BUILD_IMAGE=registry.ci.openshift.org/openshift/release:golang-1.22 \ + --opt build-arg:EL9_BUILD_IMAGE=registry.ci.openshift.org/openshift/release:golang-1.22 \ + --opt build-arg:BASE_IMAGE=quay.io/centos/centos:stream9 \ + --opt build-arg:GO="CGO_ENABLED=0 go" \ + --output type=image,name=localhost:5000/hive:latest,push=true diff --git a/hack/create_cluster_registry.sh b/hack/create_cluster_registry.sh new file mode 100755 index 0000000000..77d89cd19f --- /dev/null +++ b/hack/create_cluster_registry.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +HIVE_ROOT="$(git rev-parse --show-toplevel)" +CNI_PATH="$HIVE_ROOT"/.tmp/_output/cni/bin +export HIVE_ROOT +export CNI_PATH +export PATH=$HIVE_ROOT/.tmp/_output/bin:$PATH + +set -o errexit + +cluster_name="dev-hive" +reg_name='kind-nerdctl-registry' +reg_port='5000' + +sleep 3 + +# create cluster +cat <> "$HIVE_ROOT"/.tmp/_output/metrics.json diff --git a/hack/dev_env_vars.sh b/hack/dev_env_vars.sh new file mode 100755 index 0000000000..06188ac6cc --- /dev/null +++ b/hack/dev_env_vars.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +HIVE_ROOT="$(git rev-parse --show-toplevel)" +export HIVE_ROOT +export GOPATH="${HIVE_ROOT}/.tmp/_output/go" +export PATH=$HIVE_ROOT/.tmp/_output/bin:$HIVE_ROOT/.tmp/_output/go/bin:${GOPATH}/bin:$PATH:$PATH +export CNI_PATH=$HIVE_ROOT/.tmp/_output/cni/bin +export IMG=localhost:5000/hive:latest +export KUBECONFIG=$HIVE_ROOT/.tmp/_output/dev-hive.kubeconfig +export HIVE_OPERATOR_NS=dev-hive +export HIVE_SKIP_LEADER_ELECTION=1 +export METRICS_CONFIG_FILE=$HIVE_ROOT/.tmp/_output/metrics.json +export HIVE_NS=dev-hive +export HIVE_MACHINEPOOL_POD_NAME="hive-machinepool-0" +export HIVE_CLUSTERSYNC_POD_NAME="hive-clustersync-0" +export GOROOT=$HIVE_ROOT/.tmp/_output/go \ No newline at end of file diff --git a/hack/install_dev_env.sh b/hack/install_dev_env.sh new file mode 100755 index 0000000000..d1b52ebe1b --- /dev/null +++ b/hack/install_dev_env.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash + +# Variables +HIVE_GO_VERSION="1.22" +ROOTLESSKIT_VERSION="v2.3.1" +CONTAINERD_VERSION="2.0.1" +SLIRP4NETNS_VERSION="v1.3.1" +KUSTOMIZE_VERSION="4.1.3" +NERDCTL_VERSION="2.0.2" +CFSSL_VERSION="1.6.5" +KIND_VERSION="v0.25.0" +BUILDKIT_VERSION="v0.18.2" +CNI_PLUGINS_VERSION="v1.6.2" +RUNC_VERSION="v1.2.3" +HIVE_ROOT="$(git rev-parse --show-toplevel)" +CNI_PATH="$HIVE_ROOT"/.tmp/_output/cni/bin + +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Requirements for running this script or not available in binary form +commands=("make" "awk" "sed" "git" "python3" "pip" "iptables") + +for command in "${commands[@]}"; do + if ! command_exists "$command"; then + echo "Error: $command is not available" + exit 1 + fi +done + +export HIVE_ROOT + +mkdir -p "$HIVE_ROOT"/.tmp/_output/bin +mkdir -p "$CNI_PATH" +export CNI_PATH +export PATH="${HIVE_ROOT}/.tmp/_output/bin:$PATH" + +# Go setup +export GOPATH="${HIVE_ROOT}/.tmp/_output/go" +mkdir -p "$GOPATH" + +export PATH="${GOPATH}/bin:$PATH" + +GO_VERSION=$(go version 2>/dev/null) +CURRENT_VERSION=$(echo "$GO_VERSION" | awk '{print $3}' | sed 's/go//') + +if [[ -z "$GO_VERSION" ]] || [[ "$CURRENT_VERSION" != "${HIVE_GO_VERSION}"* ]]; then + curl -L https://go.dev/dl/go1.22.4.linux-amd64.tar.gz |\ + tar -C "$HIVE_ROOT"/.tmp/_output -xz +fi + +go install github.com/golang/mock/mockgen@latest + +# Build Hive +make + + +# Install Dependencies +if ! command_exists rootlesskit; then + curl -L "https://github.com/rootless-containers/rootlesskit/releases/download/${ROOTLESSKIT_VERSION}/rootlesskit-x86_64.tar.gz" |\ + tar -C "${HIVE_ROOT}/.tmp/_output/bin" -xz rootlesskit +fi + +# Install Containerd +if ! command_exists containerd; then + curl -L "https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-2.0.1-linux-amd64.tar.gz" |\ + tar -xz -C "${HIVE_ROOT}/.tmp/_output/bin" --strip-components=1 "bin/containerd" "bin/containerd-shim-runc-v2" +fi + +# Install Slirp4netns +if ! command_exists slirp4netns; then + curl -L "https://github.com/rootless-containers/slirp4netns/releases/download/${SLIRP4NETNS_VERSION}/slirp4netns-x86_64" -o "${HIVE_ROOT}/.tmp/_output/bin/slirp4netns" + chmod +x "${HIVE_ROOT}/.tmp/_output/bin/slirp4netns" +fi + +# Install Kustomize +if ! command_exists kustomize-4.1.3; then + curl -L https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz |\ + tar -xz -C "${HIVE_ROOT}/.tmp/_output/bin" kustomize && mv "$HIVE_ROOT"/.tmp/_output/bin/kustomize "$HIVE_ROOT"/.tmp/_output/bin/kustomize-4.1.3 +fi + +# Install nerdctl +if ! command_exists nerdctl; then + curl -L "https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz" |\ + tar -xz -C "${HIVE_ROOT}/.tmp/_output/bin" containerd-rootless.sh containerd-rootless-setuptool.sh nerdctl +fi + +# Install cfssl and cfssljson +if ! command_exists cfssl; then + curl -L "https://github.com/cloudflare/cfssl/releases/download/v${CFSSL_VERSION}/cfssl_${CFSSL_VERSION}_linux_amd64" -o "${HIVE_ROOT}/.tmp/_output/bin/cfssl" + chmod +x "${HIVE_ROOT}/.tmp/_output/bin/cfssl" +fi + +if ! command_exists cfssljson; then + curl -L "https://github.com/cloudflare/cfssl/releases/download/v${CFSSL_VERSION}/cfssljson_${CFSSL_VERSION}_linux_amd64" -o "${HIVE_ROOT}/.tmp/_output/bin/cfssljson" + chmod +x "${HIVE_ROOT}/.tmp/_output/bin/cfssljson" +fi + +# Install oc +if ! command_exists oc; then + curl -L "https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-client-linux-amd64-rhel8-4.17.0-okd-scos.0.tar.gz" |\ + tar -xz -C "${HIVE_ROOT}/.tmp/_output/bin" "oc" +fi + +# Install kubectl +if ! command_exists kubectl; then + curl -L "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -o "${HIVE_ROOT}/.tmp/_output/bin/kubectl" + chmod +x "$HIVE_ROOT"/.tmp/_output/bin/kubectl +fi + +# Install kind +if ! command_exists kind; then + curl -L "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" -o "${HIVE_ROOT}/.tmp/_output/bin/kind" + chmod +x "$HIVE_ROOT"/.tmp/_output/bin/kind +fi + +# Install buildkitd +if ! command_exists buildkitd; then + curl -L "https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz" |\ + tar -xz -C "${HIVE_ROOT}/.tmp/_output/bin" --strip-components=1 "bin/buildkitd" "bin/buildctl" +fi + +# Install CNI plugins +if [ ! -f "$CNI_PATH/bridge" ]; then + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-amd64-${CNI_PLUGINS_VERSION}.tgz" |\ + tar -xz -C "${CNI_PATH}" +fi + +# Install runc +if ! command_exists runc; then + curl -L "https://github.com/opencontainers/runc/releases/download/${RUNC_VERSION}/runc.amd64" -o "${HIVE_ROOT}/.tmp/_output/bin/runc" + chmod +x "$HIVE_ROOT"/.tmp/_output/bin/runc +fi + +# Check whether system requirements for running rootless containerd are satisfied +echo "Checking system requirements for rootless containerd..." +CHECK_OUTPUT=$(containerd-rootless-setuptool.sh check) + +if [[ "$CHECK_OUTPUT" != *"Requirements are satisfied"* ]]; then echo "Error: Requirements are not satisfied. Exiting." + exit 1 +fi + +CONTAINERD_SETUPTOOL_PATH="${HIVE_ROOT}/.tmp/_output/bin/containerd-rootless-setuptool.sh" + +# Add buildkitd flag pointing to custom /cni/bin location to containerd setup script +if ! grep -q "cni" $CONTAINERD_SETUPTOOL_PATH; then + sed -i '/BUILDKITD_FLAG="--oci-cni-binary-dir=${HIVE_ROOT}\.tmp\/_output\/cni\/bin\/"/!s/\(BUILDKITD_FLAG="--oci-worker=true --oci-worker-rootless=true --containerd-worker=false\)/\1 --oci-cni-binary-dir=\/home\/daturece\/hive\/.tmp\/_output\/cni\/bin\//g' "$CONTAINERD_SETUPTOOL_PATH" + echo "Flag added to setuptool.sh" +else + echo "Flag already exists in setuptool.sh" +fi + +# Install containerd in rootless mode, this will create a rootlesskit instance +containerd-rootless-setuptool.sh install +if [ $? -ne 0 ]; then + echo "Error: Failed to install containerd in rootless mode." + exit 1 +fi + +# Run buildkitd +containerd-rootless-setuptool.sh install-buildkit +if [ $? -ne 0 ]; then + echo "Error: Failed to install containerd in rootless mode." + exit 1 +fi + +# Check if nerdctl is running +echo "Checking nerdctl version..." +NERDCTL_VERSION_OUTPUT=$(nerdctl --version 2>/dev/null) + +if [[ "$NERDCTL_VERSION_OUTPUT" == *"nerdctl version 2.0.2"* ]]; then + echo "nerdctl is running correctly: $NERDCTL_VERSION_OUTPUT" +else + echo "Error: nerdctl is not running correctly." + exit 1 +fi diff --git a/hack/prune_dev_env.sh b/hack/prune_dev_env.sh new file mode 100755 index 0000000000..4b749d59e6 --- /dev/null +++ b/hack/prune_dev_env.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +HIVE_ROOT="$(git rev-parse --show-toplevel)" +CNI_PATH="$HIVE_ROOT"/.tmp/_output/cni/bin +export CNI_PATH +export HIVE_ROOT +export PATH=$HIVE_ROOT/.tmp/_output/bin:$PATH + +containerd_pids=$(nerdctl ps -q) + +# Stop containers +for pid in $containerd_pids; do + nerdctl stop "$pid" + nerdctl remove "$pid" +done + +# clear cache +nerdctl system prune -a -f + +# stop & uninstall rootless buildkitd +containerd-rootless-setuptool.sh uninstall-buildkit +rootlesskit rm -rf ~/.local/share/buildkit + +# stop & uninstall rootlesscontainerd +containerd-rootless-setuptool.sh uninstall +rootlesskit rm -rf ~/.local/share/containerd + +# remove dirs create by scripts +go clean -cache -modcache +rm -rf "$HIVE_ROOT"/.tmp "$HIVE_ROOT"/hiveadmission-certs \ No newline at end of file