DO NOT MERGE: Auto update cloud scanner latest

.github/workflows/build-publish-binaries.yaml

@@ -36,6 +36,7 @@ jobs:
           mkdir -p /tmp/binaries/$DF_BIN_VER
           cd /tmp/binaries/$DF_BIN_VER
           id=$(docker create deepfenceio/deepfence_agent_ce:latest)
+          cloud_id=$(docker create deepfenceio/cloud-scanner:latest)
           docker cp $id:/bin/deepfenced self
           docker cp $id:/home/deepfence/bin/package-scanner package_scanner
           docker cp $id:/home/deepfence/bin/yara-hunter/YaraHunter malware_scanner
@@ -44,6 +45,7 @@ jobs:
           docker cp $id:/opt/td-agent-bit/bin/fluent-bit fluentbit
           docker cp $id:/usr/local/bin/syft syft
           docker cp $id:/usr/local/bin/compliance_check/compliance compliance
+          docker cp $cloud_id:/home/deepfence/bin/cloud_scanner cloud_scanner
           tar zcvf binaries.tar.gz ./*
           docker rm -v $id
 

.gitmodules

@@ -6,6 +6,10 @@
 	path = deepfence_agent/plugins/package-scanner
 	url = https://github.com/deepfence/package-scanner
 	branch = main
+[submodule "deepfence_agent/plugins/cloud-scanner"]
+	path = deepfence_agent/plugins/cloud-scanner
+	url = https://github.com/deepfence/cloud-scanner
+	branch = main
 [submodule "deepfence_agent/plugins/SecretScanner"]
 	path = deepfence_agent/plugins/SecretScanner
 	url = https://github.com/deepfence/SecretScanner

Makefile

@@ -9,6 +9,7 @@ export SECRET_SCANNER_DIR=$(DEEPFENCE_AGENT_DIR)/plugins/SecretScanner
 export MALWARE_SCANNER_DIR=$(DEEPFENCE_AGENT_DIR)/plugins/YaraHunter/
 export PACKAGE_SCANNER_DIR=$(DEEPFENCE_AGENT_DIR)/plugins/package-scanner
 export COMPLIANCE_SCANNER_DIR=$(DEEPFENCE_AGENT_DIR)/plugins/compliance
+export CLOUD_SCANNER_DIR=$(DEEPFENCE_AGENT_DIR)/plugins/cloud-scanner
 export DEEPFENCE_CTL=$(PWD)/deepfence_ctl
 export DEEPFENCED=$(PWD)/deepfence_bootstrapper
 export IMAGE_REPOSITORY?=deepfenceio
@@ -46,6 +47,7 @@ bootstrap-agent-plugins:
 	(cd $(PACKAGE_SCANNER_DIR) && bash bootstrap.sh)
 	(cd $(SECRET_SCANNER_DIR) && bash bootstrap.sh)
 	(cd $(MALWARE_SCANNER_DIR) && bash bootstrap.sh)
+	(cd $(CLOUD_SCANNER_DIR) && bash bootstrap.sh)
 
 .PHONY: agent
 agent: go1_20_builder debian_builder deepfenced console_plugins

deepfence_agent/Dockerfile.cloud-agent

@@ -0,0 +1,85 @@
+ARG DF_IMG_TAG=latest
+ARG IMAGE_REPOSITORY=deepfenceio
+FROM $IMAGE_REPOSITORY/steampipe:$DF_IMG_TAG AS steampipe
+
+FROM golang:1.21-bookworm AS build
+
+WORKDIR /home/deepfence/plugins/cloud-scanner
+COPY plugins/cloud-scanner .
+RUN go build -o cloud_scanner .
+
+
+FROM debian:bookworm-slim
+MAINTAINER Deepfence Inc
+LABEL deepfence.role=system
+
+ENV CHECKPOINT_DISABLE=true \
+    DF_TLS_ON="1" \
+    MGMT_CONSOLE_PORT=443 \
+    MGMT_CONSOLE_URL_SCHEMA=https \
+    DEEPFENCE_KEY="" \
+    DF_ENABLE_CLOUD_NODE="true" \
+    HOME_DIR="/home/deepfence" \
+    COMPLIANCE_MOD_PATH="/opt/steampipe"
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends bash git ca-certificates nano logrotate sudo supervisor \
+    #&& apt-get install -y --no-install-recommends postgresql-client-15 \
+    && useradd -rm -d /home/deepfence -s /bin/bash -g root -G sudo -u 1001 deepfence \
+    && mkdir -p /opt/steampipe /var/log/deepfenced \
+    && chown deepfence /opt/steampipe
+
+# Installing FluentBit
+#RUN mkdir -p /etc/td-agent-bit/ /root/fluentbit
+#COPY tools/apache/fluentbit/* /etc/td-agent-bit/
+
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+COPY etc/fenced_logrotate.conf /etc/logrotate.d/fenced_logrotate.conf
+RUN chmod 600 /etc/logrotate.d/fenced_logrotate.conf
+
+COPY start_cloud_agent.sh /usr/local/bin/start_agent
+RUN chmod +x /usr/local/bin/start_agent
+#COPY plugins/fluent-bit/build/bin/fluent-bit /opt/td-agent-bit/bin/
+
+RUN chown -R deepfence:root /opt/ /home/deepfence /var/log /var/run/
+#    && chown -R root:root /etc/td-agent-bit/ /root/fluentbit
+
+WORKDIR /opt/steampipe
+
+USER deepfence
+COPY supervisord-cloud.conf /home/deepfence/supervisord.conf
+COPY --from=steampipe /usr/local/bin/steampipe /usr/local/bin/steampipe
+
+RUN steampipe service start \
+    && steampipe plugin install steampipe \
+    # plugin version should be in sync with Deepfence fork https://github.com/deepfence/steampipe-plugin-aws
+    && steampipe plugin install [email protected] [email protected] [email protected] [email protected] \
+    && git clone https://github.com/turbot/steampipe-mod-aws-compliance.git --branch v0.79 --depth 1 \
+    && git clone https://github.com/turbot/steampipe-mod-gcp-compliance.git --branch v0.21 --depth 1 \
+    && git clone https://github.com/turbot/steampipe-mod-azure-compliance.git --branch v0.35 --depth 1 \
+    && steampipe service stop
+
+COPY deepfenced /bin/deepfenced
+
+ENV VERSION=2.1.0 \
+    PUBLISH_CLOUD_RESOURCES_INTERVAL_MINUTES=5 \
+    FETCH_CLOUD_RESOURCES_INTERVAL_HOURS=12
+
+EXPOSE 8080
+
+COPY --from=steampipe /usr/local/bin/steampipe-plugin-aws.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/aws@latest/steampipe-plugin-aws.plugin
+COPY --from=steampipe /usr/local/bin/steampipe-plugin-gcp.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/gcp@latest/steampipe-plugin-gcp.plugin
+COPY --from=steampipe /usr/local/bin/steampipe-plugin-azure.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azure@latest/steampipe-plugin-azure.plugin
+COPY --from=steampipe /usr/local/bin/steampipe-plugin-azuread.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azuread@latest/steampipe-plugin-azuread.plugin
+
+COPY --from=build /home/deepfence/plugins/cloud-scanner/cloud_scanner /home/deepfence/bin/cloud_scanner
+
+USER root
+
+RUN ln -s /usr/local/bin/cloud_scanner /usr/local/bin/cloud_compliance_scan \
+    && apt-get -y remove git ca-certificates
+
+USER deepfence
+
+ENTRYPOINT ["/usr/local/bin/start_agent"]

deepfence_agent/build.sh

@@ -79,6 +79,15 @@ building_image(){
         exit 1
     fi
 
+    echo "Building Cloud Agent Image"
+    docker build --network host --rm=true --tag=$IMAGE_REPOSITORY/cloud-scanner:$DF_IMG_TAG -f Dockerfile.cloud-agent .
+    build_result=$?
+    if [ $build_result -ne 0 ]
+    then
+        echo "Deepfence cloud agent building failed, bailing out"
+        exit 1
+    fi
+
     echo "Building Agent Image"
     docker build --network host --rm=true --build-arg DF_IMG_TAG="${DF_IMG_TAG}" --build-arg IMAGE_REPOSITORY="${IMAGE_REPOSITORY}" --tag=$IMAGE_REPOSITORY/deepfence_agent_ce:$DF_IMG_TAG -f Dockerfile .
     build_result=$?

deepfence_agent/start_cloud_agent.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+trim() {
+  local var="$*"
+  # remove leading whitespace characters
+  var="${var#"${var%%[![:space:]]*}"}"
+  # remove trailing whitespace characters
+  var="${var%"${var##*[![:space:]]}"}"
+  echo -n "$var"
+}
+
+echoerr() { echo "$@" 1>&2; }
+
+configure_cron() {
+  #Setup cron jobs for misc tasks, it needs to be killed and restarted
+  #doesnt work smoothly inside docker!
+  sudo service cron start
+  sudo chmod 600 /etc/logrotate.d/fenced_logrotate.conf
+  (echo "*/5 * * * * /usr/sbin/logrotate /etc/logrotate.d/fenced_logrotate.conf") | sudo crontab -
+}
+
+launch_deepfenced() {
+  # In k8s, if agent pod restarts these files are not cleared
+  configure_cron
+
+  unlink /var/run/supervisor.sock 2>/dev/null
+  /usr/bin/supervisord -c /home/deepfence/supervisord.conf
+
+  echo "Starting cloud agent"
+  until [[ "$(ls /var/log/supervisor/deepfenced* 2>/dev/null | wc -w)" != "0" ]]; do
+    sleep 5
+  done
+  tail -f /var/log/supervisor/deepfenced*
+}
+
+main() {
+  sudo ln -sf bash /bin/sh
+  launch_deepfenced
+}
+
+main "$@"

deepfence_agent/supervisord-cloud.conf

@@ -0,0 +1,25 @@
+[unix_http_server]
+file=/var/run/supervisor.sock
+chmod=0700
+
+[supervisord]
+logfile=/var/log/supervisor/supervisord.log
+pidfile=/var/run/supervisord.pid
+childlogdir=/var/log/supervisor
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:deepfenced]
+environment=DF_ENABLE_CLOUD_NODE=true
+command=/bin/deepfenced
+numprocs=1
+autostart=true
+autorestart=true
+startsecs=0
+stopwaitsecs=30
+killasgroup=true
+priority=50

deepfence_bootstrapper/assets/config-cloud.ini

@@ -0,0 +1,6 @@
+[process:cloud_scanner]
+command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/cloud-scanner.sock && $DF_INSTALL_DIR/home/deepfence/bin/cloud_scanner -mgmt-console-url $MGMT_CONSOLE_URL -mgmt-console-port $MGMT_CONSOLE_PORT -deepfence-key $DEEPFENCE_KEY -socket-path $DF_INSTALL_DIR/tmp/cloud-scanner.sock"
+path=$DF_INSTALL_DIR/home/deepfence/bin/cloud_scanner
+autostart=true
+autorestart=true
+environment=DF_MULTI_ACC_ID="$DF_MULTI_ACC_ID",DF_ORG_ACC_ID="$DF_ORG_ACC_ID",DF_HOST_ID=$DF_HOST_ID

deepfence_bootstrapper/controls/controls.go

@@ -190,3 +190,54 @@ func SetAgentControls() {
 		log.Error().Err(err).Msg("set controls")
 	}
 }
+
+func SetCloudScannerControls() {
+	err := router.RegisterControl(ctl.StartCloudComplianceScan,
+		router.StartCloudComplianceScan)
+	if err != nil {
+		log.Error().Msgf("set controls: %v", err)
+	}
+
+	err = router.RegisterControl(ctl.StopCloudComplianceScan,
+		router.StopCloudComplianceScan)
+	if err != nil {
+		log.Error().Msgf("set controls: %v", err)
+	}
+
+	err = router.RegisterControl(ctl.RefreshResources,
+		router.RefreshResources)
+	if err != nil {
+		log.Error().Msgf("set controls: %v", err)
+	}
+	err = router.RegisterControl(ctl.StartAgentUpgrade,
+		func(req ctl.StartAgentUpgradeRequest) error {
+			log.Info().Msg("Start Agent Upgrade")
+			router.SetUpgrade()
+			return router.StartAgentUpgrade(req)
+		})
+	if err != nil {
+		log.Error().Msgf("set controls: %v", err)
+	}
+	err = router.RegisterControl(ctl.StartAgentPlugin,
+		func(req ctl.EnableAgentPluginRequest) error {
+			log.Info().Msg("Start & download Agent Plugin")
+			router.SetUpgrade()
+			err = supervisor.UpgradeProcessFromURL(req.PluginName, req.BinURL)
+			if err != nil {
+				return err
+			}
+			return supervisor.StartProcess(req.PluginName)
+		})
+	if err != nil {
+		log.Error().Msgf("set controls: %v", err)
+	}
+	err = router.RegisterControl(ctl.StopAgentPlugin,
+		func(req ctl.DisableAgentPluginRequest) error {
+			log.Info().Msg("Stop Agent Plugin")
+			return supervisor.StopProcess(req.PluginName)
+		})
+	if err != nil {
+		log.Error().Msgf("set controls: %v", err)
+	}
+
+}

deepfence_bootstrapper/controls/openapi_client_controls.go

@@ -92,7 +92,7 @@ func (ct *ControlsClient) API() *client.APIClient {
 	return ct.client.Client()
 }
 
-func (ct *ControlsClient) StartControlsWatching(nodeID string, isClusterAgent bool) error {
+func (ct *ControlsClient) StartControlsWatching(nodeID string, isClusterAgent bool, nodeType string) error {
 	if isClusterAgent {
 
 	} else {
@@ -101,6 +101,7 @@ func (ct *ControlsClient) StartControlsWatching(nodeID string, isClusterAgent bo
 			*client.NewModelInitAgentReq(
 				getMaxAllocatable(),
 				nodeID,
+				nodeType,
 				version,
 			),
 		)

deepfence_bootstrapper/go.mod

@@ -14,9 +14,11 @@ replace github.com/deepfence/agent-plugins-grpc => ../deepfence_agent/plugins/ag
 
 replace github.com/deepfence/compliance => ../deepfence_agent/plugins/compliance
 
+replace github.com/deepfence/cloud-scanner => ../deepfence_agent/plugins/cloud-scanner
+
 require (
 	github.com/containerd/cgroups/v3 v3.0.2
-	github.com/deepfence/ThreatMapper/deepfence_utils v0.0.0-00010101000000-000000000000
+	github.com/deepfence/ThreatMapper/deepfence_utils v0.0.0-20231128212448-303e0bc26021
 	github.com/deepfence/agent-plugins-grpc v1.1.0
 	github.com/deepfence/compliance v0.0.0-00010101000000-000000000000
 	github.com/deepfence/df-utils v0.0.0-00010101000000-000000000000

deepfence_bootstrapper/go.sum

@@ -134,8 +134,7 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
-github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
 github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=

deepfence_bootstrapper/main.go

@@ -16,7 +16,6 @@ import (
 	"github.com/deepfence/ThreatMapper/deepfence_bootstrapper/server"
 	"github.com/deepfence/ThreatMapper/deepfence_bootstrapper/supervisor"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
-
 	dfUtils "github.com/deepfence/df-utils"
 )
 
@@ -28,8 +27,13 @@ var configFile []byte
 //go:embed assets/config-cluster.ini
 var configClusterFile []byte
 
+//go:embed assets/config-cloud.ini
+var configCloudFile []byte
+
 var enableClusterDiscovery bool
 
+var enableCloudNode bool
+
 var enableDebug bool
 
 var hostname string
@@ -39,12 +43,20 @@ var Version string
 func init() {
 	var err error
 	enableClusterDiscovery = os.Getenv("DF_ENABLE_CLUSTER_DISCOVERY") != ""
+	enableCloudNode = os.Getenv("DF_ENABLE_CLOUD_NODE") != ""
 	if hostname = os.Getenv("SCOPE_HOSTNAME"); hostname == "" {
 		hostname, err = os.Hostname()
 		if err != nil {
 			hostname = "(unknown)"
 		}
 	}
+	if enableCloudNode {
+		hostname = "cloud-agent-" + hostname
+	}
+	err = os.Setenv("DF_HOST_ID", hostname)
+	if err != nil {
+		log.Error().Msgf("Failed to set DF_HOST_ID: %v", err)
+	}
 
 	verbosity := "info"
 	enableDebug = os.Getenv("DF_ENABLE_DEBUG") != ""
@@ -65,6 +77,8 @@ func main() {
 	var err error
 	if enableClusterDiscovery {
 		cfg, err = config.NewIniConfig(configClusterFile)
+	} else if enableCloudNode {
+		cfg, err = config.NewIniConfig(configCloudFile)
 	} else {
 		cfg, err = config.NewIniConfig(configFile)
 	}
@@ -107,6 +121,8 @@ func main() {
 	if enableClusterDiscovery {
 		_, k8sClusterName, _, _, _ := dfUtils.GetKubernetesDetails()
 		controls.SetClusterAgentControls(k8sClusterName)
+	} else if enableCloudNode {
+		controls.SetCloudScannerControls()
 	} else {
 		controls.SetAgentControls()
 	}
@@ -127,10 +143,13 @@ func main() {
 	for {
 		if enableClusterDiscovery {
 			k8sClusterID, _, _, _, _ := dfUtils.GetKubernetesDetails()
-			err = consoleClient.StartControlsWatching(k8sClusterID, true, Version)
+			err = consoleClient.StartControlsWatching(k8sClusterID, true, Version, "k8s")
 			log.Info().Msgf("cluster agent mode: %s", k8sClusterID)
+		} else if enableCloudNode {
+			err = consoleClient.StartControlsWatching(hostname, false, Version, "cloud_agent")
+			log.Info().Msgf("cloud agent mode: %s", hostname)
 		} else {
-			err = consoleClient.StartControlsWatching(hostname, false, Version)
+			err = consoleClient.StartControlsWatching(hostname, false, Version, "host")
 			log.Info().Msgf("regular agent mode: %s", hostname)
 		}
 		if err == nil {